00000080:00200000:3.0:1787827173.126966:0:27439:0:(file.c:2207:ll_file_io_generic()) Range unlock [420984, 420984] 00000080:00200000:3.0:1787827173.126974:0:27439:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787827173.126979:0:27439:0:(file.c:2240:ll_file_io_generic()) oosfile: 2 io complete with rc: 0, result: 1024, restart: 0 00000080:00200000:3.0:1787827173.126986:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724351488 i_blocks 3365288 00000080:00200000:3.0:1787827173.126992:0:27439:0:(file.c:2326:ll_file_io_generic()) iot: 2, result: 1024 00000080:00200008:3.0:1787827173.127003:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724351488, count: 1024 00000080:00200000:3.0:1787827173.127011:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 420984 of 1024 to 1024 bytes 00000080:00200000:3.0:1787827173.127016:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724351488, len 1024, copied 1024 00000080:00200000:3.0:1787827173.127025:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724352512 i_blocks 3365288 00000080:00200000:3.0:1787827173.127033:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.127040:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724352512, count: 1024 00000080:00200000:3.0:1787827173.127044:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 420984 of 2048 to 1024 bytes 00000080:00200000:3.0:1787827173.127048:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724352512, len 1024, copied 1024 00000080:00200000:3.0:1787827173.127055:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724353536 i_blocks 3365288 00000080:00200000:3.0:1787827173.127056:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.127059:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724353536, count: 1024 00000080:00200000:3.0:1787827173.127062:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 420984 of 3072 to 1024 bytes 00000080:00200000:3.0:1787827173.127064:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724353536, len 1024, copied 1024 00000080:00200000:3.0:1787827173.127067:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724354560 i_blocks 3365288 00000080:00200000:3.0:1787827173.127073:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.127076:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724354560, count: 1024 00000080:00200000:3.0:1787827173.127080:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 420985 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.127084:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 0, original count 1024 00000080:00200000:3.0:1787827173.127089:0:27439:0:(file.c:2090:ll_file_io_generic()) oosfile: write ppos: 1724354560, bytes: 1024 00000080:00200000:3.0:1787827173.127095:0:27439:0:(file.c:1848:ll_io_set_mirror()) oosfile: desiginated mirror: 0 00000020:00200000:3.0:1787827173.127103:0:27439:0:(cl_io.c:218:cl_io_rw_init()) header@000000007788600a[0x110, 368538, [0x200000402:0x27:0x0] hash] 00000020:00200000:3.0:1787827173.127107:0:27439:0:(cl_io.c:218:cl_io_rw_init()) io range: 2 [1724354560, 1724355584) 0 0 00000080:00200000:3.0:1787827173.127111:0:27439:0:(vvp_io.c:1889:vvp_io_init()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1787827173.127121:0:27439:0:(file.c:2170:ll_file_io_generic()) Range lock [420985, 420985] 00020000:00200000:3.0:1787827173.127125:0:27439:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 411 chunk: [1724354560, 1724355584) 1724355584, 1024 00020000:00200000:3.0:1787827173.127129:0:27439:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1787827173.127137:0:27439:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [420985, 420985] 00000008:00010000:3.0:1787827173.127146:0:27439:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1787827173.127150:0:27439:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff9e35c7d81d30, osc lock ffff9e35c2ddcc30, flags 0x0 00010000:00010000:3.0:1787827173.127164:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.127186:0:27439:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (860327936 860332031) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.127202:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 20/0,2 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.127223:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 20/0,2 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.127247:0:27439:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 19/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.127271:0:27439:0:(vvp_io.c:1339:vvp_io_write_start()) oosfile: write [1724354560, 1724355584) append 0 append lockpos 0 00000080:00200000:3.0:1787827173.127279:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 420985 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.127298:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724354560, len 1024, copied 1024 00000080:00200000:3.0:1787827173.127306:0:27439:0:(rw26.c:933:ll_write_end()) page@000000008fc8ec26[3 000000005ce0a915 2 1 00000000a23cbd2a 1 0 420985] 00000080:00200000:3.0:1787827173.127310:0:27439:0:(rw26.c:933:ll_write_end()) vmpage @00000000c7a572af fffffc0001001 3:0 ffff9e34c3d8f828 420985 lru 00000080:00200000:3.0:1787827173.127314:0:27439:0:(rw26.c:933:ll_write_end()) osc-page@00000000c7f24d1e 210041: 1< 0 - - > 2< 860327936 0 0 0x0 0x0 | 000000004db7037e 0000000052729811 > 3< 0 0 > 4< 0 0 8 484118528 - | - - + - > 5< - - + - | 0 - | 339 - -> 00000080:00200000:3.0:1787827173.127316:0:27439:0:(rw26.c:933:ll_write_end()) end page@000000008fc8ec26 00000080:00200000:3.0:1787827173.127318:0:27439:0:(rw26.c:933:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1787827173.127320:0:27439:0:(vvp_io.c:1214:vvp_io_write_commit()) commit async pages: 1, from 0, to 1024 prio 0 00000008:00010000:3.0:1787827173.127328:0:27439:0:(osc_cache.c:565:osc_extent_merge()) ### extent: 000000001063fa34 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.127343:0:27439:0:(osc_cache.c:497:__osc_extent_remove()) ### extent: 000000001063fa34 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.127351:0:27439:0:(osc_cache.c:582:osc_extent_merge()) ### extent: 00000000df269729 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.127357:0:27439:0:(osc_cache.c:846:osc_extent_find()) ### extent: 00000000df269729 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.127374:0:27439:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff9e35d0e392f8, count 1, dirtied 1 00000080:00200000:3.0:1787827173.127379:0:27439:0:(vvp_io.c:1237:vvp_io_write_commit()) Committed 1 pages 1024 bytes, tot: 1024 00000080:00200000:3.0:1787827173.127386:0:27439:0:(vvp_io.c:1421:vvp_io_write_start()) oosfile: write bytes 0, result: 1024 00000020:00010000:3.0:1787827173.127395:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_release():245 00000020:00010000:3.0:1787827173.127400:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1787827173.127408:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c7d81d30 (ffff9e35da704cf8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1787827173.127413:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.127425:0:27439:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 17/0,0 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1787827173.127441:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_fini():72 00000020:00010000:3.0:1787827173.127450:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c7d81d30 (ffff9e35c614ce18/1) at cl_lock_fini():72 00000080:00200000:3.0:1787827173.127460:0:27439:0:(vvp_io.c:533:vvp_io_advance()) advancing 1024 bytes 00000080:00200000:3.0:1787827173.127466:0:27439:0:(file.c:2207:ll_file_io_generic()) Range unlock [420985, 420985] 00000080:00200000:3.0:1787827173.127475:0:27439:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787827173.127479:0:27439:0:(file.c:2240:ll_file_io_generic()) oosfile: 2 io complete with rc: 0, result: 1024, restart: 0 00000080:00200000:3.0:1787827173.127485:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724355584 i_blocks 3365288 00000080:00200000:3.0:1787827173.127490:0:27439:0:(file.c:2326:ll_file_io_generic()) iot: 2, result: 1024 00000080:00200008:3.0:1787827173.127501:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724355584, count: 1024 00000080:00200000:3.0:1787827173.127511:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 420985 of 1024 to 1024 bytes 00000080:00200000:3.0:1787827173.127518:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724355584, len 1024, copied 1024 00000080:00200000:3.0:1787827173.127526:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724356608 i_blocks 3365288 00000080:00200000:3.0:1787827173.127532:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.127540:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724356608, count: 1024 00000080:00200000:3.0:1787827173.127546:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 420985 of 2048 to 1024 bytes 00000080:00200000:3.0:1787827173.127550:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724356608, len 1024, copied 1024 00000080:00200000:3.0:1787827173.127555:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724357632 i_blocks 3365288 00000080:00200000:3.0:1787827173.127557:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.127559:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724357632, count: 1024 00000080:00200000:3.0:1787827173.127562:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 420985 of 3072 to 1024 bytes 00000080:00200000:3.0:1787827173.127564:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724357632, len 1024, copied 1024 00000080:00200000:3.0:1787827173.127567:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724358656 i_blocks 3365288 00000080:00200000:3.0:1787827173.127569:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.127573:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724358656, count: 1024 00000080:00200000:3.0:1787827173.127578:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 420986 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.127580:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 0, original count 1024 00000080:00200000:3.0:1787827173.127583:0:27439:0:(file.c:2090:ll_file_io_generic()) oosfile: write ppos: 1724358656, bytes: 1024 00000080:00200000:3.0:1787827173.127588:0:27439:0:(file.c:1848:ll_io_set_mirror()) oosfile: desiginated mirror: 0 00000020:00200000:3.0:1787827173.127596:0:27439:0:(cl_io.c:218:cl_io_rw_init()) header@000000007788600a[0x110, 368539, [0x200000402:0x27:0x0] hash] 00000020:00200000:3.0:1787827173.127597:0:27439:0:(cl_io.c:218:cl_io_rw_init()) io range: 2 [1724358656, 1724359680) 0 0 00000080:00200000:3.0:1787827173.127602:0:27439:0:(vvp_io.c:1889:vvp_io_init()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1787827173.127612:0:27439:0:(file.c:2170:ll_file_io_generic()) Range lock [420986, 420986] 00020000:00200000:3.0:1787827173.127619:0:27439:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 411 chunk: [1724358656, 1724359680) 1724359680, 1024 00020000:00200000:3.0:1787827173.127624:0:27439:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1787827173.127631:0:27439:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [420986, 420986] 00000008:00010000:3.0:1787827173.127639:0:27439:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1787827173.127643:0:27439:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff9e35c7d81d30, osc lock ffff9e35c2ddcc30, flags 0x0 00010000:00010000:3.0:1787827173.127655:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.127678:0:27439:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (860332032 860336127) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.127703:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 20/0,2 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.127715:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 20/0,2 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.127727:0:27439:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 19/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.127743:0:27439:0:(vvp_io.c:1339:vvp_io_write_start()) oosfile: write [1724358656, 1724359680) append 0 append lockpos 0 00000080:00200000:3.0:1787827173.127750:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 420986 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.127771:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724358656, len 1024, copied 1024 00000080:00200000:3.0:1787827173.127780:0:27439:0:(rw26.c:933:ll_write_end()) page@00000000a6a06c8a[3 000000005ce0a915 2 1 00000000a23cbd2a 1 0 420986] 00000080:00200000:3.0:1787827173.127786:0:27439:0:(rw26.c:933:ll_write_end()) vmpage @00000000c7bc3bc2 fffffc0001001 3:0 ffff9e34c3d8f570 420986 lru 00000080:00200000:3.0:1787827173.127822:0:27439:0:(rw26.c:933:ll_write_end()) osc-page@000000003e0f8496 210042: 1< 0 - - > 2< 860332032 0 0 0x0 0x0 | 000000004db7037e 0000000052729811 > 3< 0 0 > 4< 0 0 8 484114432 - | - - + - > 5< - - + - | 0 - | 340 - -> 00000080:00200000:3.0:1787827173.127828:0:27439:0:(rw26.c:933:ll_write_end()) end page@00000000a6a06c8a 00000080:00200000:3.0:1787827173.127831:0:27439:0:(rw26.c:933:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1787827173.127836:0:27439:0:(vvp_io.c:1214:vvp_io_write_commit()) commit async pages: 1, from 0, to 1024 prio 0 00000008:00010000:3.0:1787827173.127847:0:27439:0:(osc_cache.c:565:osc_extent_merge()) ### extent: 000000001063fa34 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.127868:0:27439:0:(osc_cache.c:497:__osc_extent_remove()) ### extent: 000000001063fa34 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.127881:0:27439:0:(osc_cache.c:582:osc_extent_merge()) ### extent: 00000000df269729 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.127897:0:27439:0:(osc_cache.c:846:osc_extent_find()) ### extent: 00000000df269729 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.127937:0:27439:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff9e35d0e392f8, count 1, dirtied 1 00000080:00200000:3.0:1787827173.127943:0:27439:0:(vvp_io.c:1237:vvp_io_write_commit()) Committed 1 pages 1024 bytes, tot: 1024 00000080:00200000:3.0:1787827173.127948:0:27439:0:(vvp_io.c:1421:vvp_io_write_start()) oosfile: write bytes 0, result: 1024 00000020:00010000:3.0:1787827173.127960:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_release():245 00000020:00010000:3.0:1787827173.127968:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1787827173.127977:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c7d81d30 (ffff9e35da704cf8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1787827173.127987:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.128005:0:27439:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 17/0,0 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1787827173.128021:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_fini():72 00000020:00010000:3.0:1787827173.128026:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c7d81d30 (ffff9e35c614ce18/1) at cl_lock_fini():72 00000080:00200000:3.0:1787827173.128036:0:27439:0:(vvp_io.c:533:vvp_io_advance()) advancing 1024 bytes 00000080:00200000:3.0:1787827173.128041:0:27439:0:(file.c:2207:ll_file_io_generic()) Range unlock [420986, 420986] 00000080:00200000:3.0:1787827173.128049:0:27439:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787827173.128059:0:27439:0:(file.c:2240:ll_file_io_generic()) oosfile: 2 io complete with rc: 0, result: 1024, restart: 0 00000080:00200000:3.0:1787827173.128068:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724359680 i_blocks 3365288 00000080:00200000:3.0:1787827173.128074:0:27439:0:(file.c:2326:ll_file_io_generic()) iot: 2, result: 1024 00000080:00200008:3.0:1787827173.128085:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724359680, count: 1024 00000080:00200000:3.0:1787827173.128093:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 420986 of 1024 to 1024 bytes 00000080:00200000:3.0:1787827173.128099:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724359680, len 1024, copied 1024 00000080:00200000:3.0:1787827173.128104:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724360704 i_blocks 3365288 00000080:00200000:3.0:1787827173.128106:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.128111:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724360704, count: 1024 00000080:00200000:3.0:1787827173.128115:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 420986 of 2048 to 1024 bytes 00000080:00200000:3.0:1787827173.128121:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724360704, len 1024, copied 1024 00000080:00200000:3.0:1787827173.128128:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724361728 i_blocks 3365288 00000080:00200000:3.0:1787827173.128133:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.128141:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724361728, count: 1024 00000080:00200000:3.0:1787827173.128149:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 420986 of 3072 to 1024 bytes 00000080:00200000:3.0:1787827173.128154:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724361728, len 1024, copied 1024 00000080:00200000:3.0:1787827173.128159:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724362752 i_blocks 3365288 00000080:00200000:3.0:1787827173.128166:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.128173:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724362752, count: 1024 00000080:00200000:3.0:1787827173.128182:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 420987 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.128187:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 0, original count 1024 00000080:00200000:3.0:1787827173.128192:0:27439:0:(file.c:2090:ll_file_io_generic()) oosfile: write ppos: 1724362752, bytes: 1024 00000080:00200000:3.0:1787827173.128200:0:27439:0:(file.c:1848:ll_io_set_mirror()) oosfile: desiginated mirror: 0 00000020:00200000:3.0:1787827173.128206:0:27439:0:(cl_io.c:218:cl_io_rw_init()) header@000000007788600a[0x110, 368540, [0x200000402:0x27:0x0] hash] 00000020:00200000:3.0:1787827173.128208:0:27439:0:(cl_io.c:218:cl_io_rw_init()) io range: 2 [1724362752, 1724363776) 0 0 00000080:00200000:3.0:1787827173.128214:0:27439:0:(vvp_io.c:1889:vvp_io_init()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1787827173.128224:0:27439:0:(file.c:2170:ll_file_io_generic()) Range lock [420987, 420987] 00020000:00200000:3.0:1787827173.128229:0:27439:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 411 chunk: [1724362752, 1724363776) 1724363776, 1024 00020000:00200000:3.0:1787827173.128236:0:27439:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1787827173.128245:0:27439:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [420987, 420987] 00000008:00010000:3.0:1787827173.128252:0:27439:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1787827173.128256:0:27439:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff9e35c7d81d30, osc lock ffff9e35c2ddcc30, flags 0x0 00010000:00010000:3.0:1787827173.128269:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.128298:0:27439:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (860336128 860340223) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.128318:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 20/0,2 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.128333:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 20/0,2 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.128349:0:27439:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 19/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.128359:0:27439:0:(vvp_io.c:1339:vvp_io_write_start()) oosfile: write [1724362752, 1724363776) append 0 append lockpos 0 00000080:00200000:3.0:1787827173.128368:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 420987 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.128390:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724362752, len 1024, copied 1024 00000080:00200000:3.0:1787827173.128402:0:27439:0:(rw26.c:933:ll_write_end()) page@00000000df1d52ef[3 000000005ce0a915 2 1 00000000a23cbd2a 1 0 420987] 00000080:00200000:3.0:1787827173.128407:0:27439:0:(rw26.c:933:ll_write_end()) vmpage @0000000068b44cdd fffffc0001001 3:0 ffff9e34c3d8fe80 420987 lru 00000080:00200000:3.0:1787827173.128418:0:27439:0:(rw26.c:933:ll_write_end()) osc-page@00000000d0008dfa 210043: 1< 0 - - > 2< 860336128 0 0 0x0 0x0 | 000000004db7037e 0000000052729811 > 3< 0 0 > 4< 0 0 8 484110336 - | - - + - > 5< - - + - | 0 - | 341 - -> 00000080:00200000:3.0:1787827173.128423:0:27439:0:(rw26.c:933:ll_write_end()) end page@00000000df1d52ef 00000080:00200000:3.0:1787827173.128427:0:27439:0:(rw26.c:933:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1787827173.128430:0:27439:0:(vvp_io.c:1214:vvp_io_write_commit()) commit async pages: 1, from 0, to 1024 prio 0 00000008:00010000:3.0:1787827173.128443:0:27439:0:(osc_cache.c:565:osc_extent_merge()) ### extent: 000000001063fa34 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.128472:0:27439:0:(osc_cache.c:497:__osc_extent_remove()) ### extent: 000000001063fa34 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.128489:0:27439:0:(osc_cache.c:582:osc_extent_merge()) ### extent: 00000000df269729 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.128502:0:27439:0:(osc_cache.c:846:osc_extent_find()) ### extent: 00000000df269729 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.128520:0:27439:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff9e35d0e392f8, count 1, dirtied 1 00000080:00200000:3.0:1787827173.128526:0:27439:0:(vvp_io.c:1237:vvp_io_write_commit()) Committed 1 pages 1024 bytes, tot: 1024 00000080:00200000:3.0:1787827173.128531:0:27439:0:(vvp_io.c:1421:vvp_io_write_start()) oosfile: write bytes 0, result: 1024 00000020:00010000:3.0:1787827173.128542:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_release():245 00000020:00010000:3.0:1787827173.128547:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1787827173.128551:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c7d81d30 (ffff9e35da704cf8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1787827173.128560:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.128573:0:27439:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 17/0,0 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1787827173.128593:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_fini():72 00000020:00010000:3.0:1787827173.128597:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c7d81d30 (ffff9e35c614ce18/1) at cl_lock_fini():72 00000080:00200000:3.0:1787827173.128606:0:27439:0:(vvp_io.c:533:vvp_io_advance()) advancing 1024 bytes 00000080:00200000:3.0:1787827173.128613:0:27439:0:(file.c:2207:ll_file_io_generic()) Range unlock [420987, 420987] 00000080:00200000:3.0:1787827173.128618:0:27439:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787827173.128626:0:27439:0:(file.c:2240:ll_file_io_generic()) oosfile: 2 io complete with rc: 0, result: 1024, restart: 0 00000080:00200000:3.0:1787827173.128635:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724363776 i_blocks 3365288 00000080:00200000:3.0:1787827173.128641:0:27439:0:(file.c:2326:ll_file_io_generic()) iot: 2, result: 1024 00000080:00200008:3.0:1787827173.128652:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724363776, count: 1024 00000080:00200000:3.0:1787827173.128663:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 420987 of 1024 to 1024 bytes 00000080:00200000:3.0:1787827173.128670:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724363776, len 1024, copied 1024 00000080:00200000:3.0:1787827173.128677:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724364800 i_blocks 3365288 00000080:00200000:3.0:1787827173.128679:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.128684:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724364800, count: 1024 00000080:00200000:3.0:1787827173.128688:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 420987 of 2048 to 1024 bytes 00000080:00200000:3.0:1787827173.128692:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724364800, len 1024, copied 1024 00000080:00200000:3.0:1787827173.128698:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724365824 i_blocks 3365288 00000080:00200000:3.0:1787827173.128706:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.128714:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724365824, count: 1024 00000080:00200000:3.0:1787827173.128720:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 420987 of 3072 to 1024 bytes 00000080:00200000:3.0:1787827173.128725:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724365824, len 1024, copied 1024 00000080:00200000:3.0:1787827173.128732:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724366848 i_blocks 3365288 00000080:00200000:3.0:1787827173.128738:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.128746:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724366848, count: 1024 00000080:00200000:3.0:1787827173.128752:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 420988 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.128755:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 0, original count 1024 00000080:00200000:3.0:1787827173.128758:0:27439:0:(file.c:2090:ll_file_io_generic()) oosfile: write ppos: 1724366848, bytes: 1024 00000080:00200000:3.0:1787827173.128763:0:27439:0:(file.c:1848:ll_io_set_mirror()) oosfile: desiginated mirror: 0 00000020:00200000:3.0:1787827173.128773:0:27439:0:(cl_io.c:218:cl_io_rw_init()) header@000000007788600a[0x110, 368541, [0x200000402:0x27:0x0] hash] 00000020:00200000:3.0:1787827173.128775:0:27439:0:(cl_io.c:218:cl_io_rw_init()) io range: 2 [1724366848, 1724367872) 0 0 00000080:00200000:3.0:1787827173.128780:0:27439:0:(vvp_io.c:1889:vvp_io_init()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1787827173.128788:0:27439:0:(file.c:2170:ll_file_io_generic()) Range lock [420988, 420988] 00020000:00200000:3.0:1787827173.128793:0:27439:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 411 chunk: [1724366848, 1724367872) 1724367872, 1024 00020000:00200000:3.0:1787827173.128827:0:27439:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1787827173.128837:0:27439:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [420988, 420988] 00000008:00010000:3.0:1787827173.128847:0:27439:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1787827173.128851:0:27439:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff9e35c7d81d30, osc lock ffff9e35c2ddcc30, flags 0x0 00010000:00010000:3.0:1787827173.128862:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.128875:0:27439:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (860340224 860344319) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.128894:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 20/0,2 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.128931:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 20/0,2 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.128953:0:27439:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 19/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.128973:0:27439:0:(vvp_io.c:1339:vvp_io_write_start()) oosfile: write [1724366848, 1724367872) append 0 append lockpos 0 00000080:00200000:3.0:1787827173.128983:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 420988 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.129004:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724366848, len 1024, copied 1024 00000080:00200000:3.0:1787827173.129013:0:27439:0:(rw26.c:933:ll_write_end()) page@0000000022c4aeb2[3 000000005ce0a915 2 1 00000000a23cbd2a 1 0 420988] 00000080:00200000:3.0:1787827173.129018:0:27439:0:(rw26.c:933:ll_write_end()) vmpage @00000000f2942b04 fffffc0001001 3:0 ffff9e34c3d8f740 420988 lru 00000080:00200000:3.0:1787827173.129030:0:27439:0:(rw26.c:933:ll_write_end()) osc-page@00000000be5920c8 210044: 1< 0 - - > 2< 860340224 0 0 0x0 0x0 | 000000004db7037e 0000000052729811 > 3< 0 0 > 4< 0 0 8 484106240 - | - - + - > 5< - - + - | 0 - | 342 - -> 00000080:00200000:3.0:1787827173.129034:0:27439:0:(rw26.c:933:ll_write_end()) end page@0000000022c4aeb2 00000080:00200000:3.0:1787827173.129037:0:27439:0:(rw26.c:933:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1787827173.129042:0:27439:0:(vvp_io.c:1214:vvp_io_write_commit()) commit async pages: 1, from 0, to 1024 prio 0 00000008:00010000:3.0:1787827173.129053:0:27439:0:(osc_cache.c:565:osc_extent_merge()) ### extent: 000000001063fa34 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.129078:0:27439:0:(osc_cache.c:497:__osc_extent_remove()) ### extent: 000000001063fa34 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.129089:0:27439:0:(osc_cache.c:582:osc_extent_merge()) ### extent: 00000000df269729 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.129106:0:27439:0:(osc_cache.c:846:osc_extent_find()) ### extent: 00000000df269729 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.129131:0:27439:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff9e35d0e392f8, count 1, dirtied 1 00000080:00200000:3.0:1787827173.129137:0:27439:0:(vvp_io.c:1237:vvp_io_write_commit()) Committed 1 pages 1024 bytes, tot: 1024 00000080:00200000:3.0:1787827173.129143:0:27439:0:(vvp_io.c:1421:vvp_io_write_start()) oosfile: write bytes 0, result: 1024 00000020:00010000:3.0:1787827173.129153:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_release():245 00000020:00010000:3.0:1787827173.129158:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1787827173.129166:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c7d81d30 (ffff9e35da704cf8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1787827173.129172:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.129184:0:27439:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 17/0,0 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1787827173.129204:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_fini():72 00000020:00010000:3.0:1787827173.129211:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c7d81d30 (ffff9e35c614ce18/1) at cl_lock_fini():72 00000080:00200000:3.0:1787827173.129218:0:27439:0:(vvp_io.c:533:vvp_io_advance()) advancing 1024 bytes 00000080:00200000:3.0:1787827173.129225:0:27439:0:(file.c:2207:ll_file_io_generic()) Range unlock [420988, 420988] 00000080:00200000:3.0:1787827173.129232:0:27439:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787827173.129236:0:27439:0:(file.c:2240:ll_file_io_generic()) oosfile: 2 io complete with rc: 0, result: 1024, restart: 0 00000080:00200000:3.0:1787827173.129242:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724367872 i_blocks 3365288 00000080:00200000:3.0:1787827173.129246:0:27439:0:(file.c:2326:ll_file_io_generic()) iot: 2, result: 1024 00000080:00200008:3.0:1787827173.129258:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724367872, count: 1024 00000080:00200000:3.0:1787827173.129268:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 420988 of 1024 to 1024 bytes 00000080:00200000:3.0:1787827173.129274:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724367872, len 1024, copied 1024 00000080:00200000:3.0:1787827173.129283:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724368896 i_blocks 3365288 00000080:00200000:3.0:1787827173.129290:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.129296:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724368896, count: 1024 00000080:00200000:3.0:1787827173.129303:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 420988 of 2048 to 1024 bytes 00000080:00200000:3.0:1787827173.129307:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724368896, len 1024, copied 1024 00000080:00200000:3.0:1787827173.129313:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724369920 i_blocks 3365288 00000080:00200000:3.0:1787827173.129317:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.129323:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724369920, count: 1024 00000080:00200000:3.0:1787827173.129330:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 420988 of 3072 to 1024 bytes 00000080:00200000:3.0:1787827173.129332:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724369920, len 1024, copied 1024 00000080:00200000:3.0:1787827173.129337:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724370944 i_blocks 3365288 00000080:00200000:3.0:1787827173.129343:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.129347:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724370944, count: 1024 00000080:00200000:3.0:1787827173.129352:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 420989 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.129355:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 0, original count 1024 00000080:00200000:3.0:1787827173.129358:0:27439:0:(file.c:2090:ll_file_io_generic()) oosfile: write ppos: 1724370944, bytes: 1024 00000080:00200000:3.0:1787827173.129361:0:27439:0:(file.c:1848:ll_io_set_mirror()) oosfile: desiginated mirror: 0 00000020:00200000:3.0:1787827173.129370:0:27439:0:(cl_io.c:218:cl_io_rw_init()) header@000000007788600a[0x110, 368542, [0x200000402:0x27:0x0] hash] 00000020:00200000:3.0:1787827173.129374:0:27439:0:(cl_io.c:218:cl_io_rw_init()) io range: 2 [1724370944, 1724371968) 0 0 00000080:00200000:3.0:1787827173.129380:0:27439:0:(vvp_io.c:1889:vvp_io_init()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1787827173.129388:0:27439:0:(file.c:2170:ll_file_io_generic()) Range lock [420989, 420989] 00020000:00200000:3.0:1787827173.129393:0:27439:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 411 chunk: [1724370944, 1724371968) 1724371968, 1024 00020000:00200000:3.0:1787827173.129399:0:27439:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1787827173.129406:0:27439:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [420989, 420989] 00000008:00010000:3.0:1787827173.129416:0:27439:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1787827173.129421:0:27439:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff9e35c7d81d30, osc lock ffff9e35c2ddcc30, flags 0x0 00010000:00010000:3.0:1787827173.129433:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.129446:0:27439:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (860344320 860348415) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.129462:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 20/0,2 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.129475:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 20/0,2 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.129488:0:27439:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 19/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.129500:0:27439:0:(vvp_io.c:1339:vvp_io_write_start()) oosfile: write [1724370944, 1724371968) append 0 append lockpos 0 00000080:00200000:3.0:1787827173.129509:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 420989 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.129529:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724370944, len 1024, copied 1024 00000080:00200000:3.0:1787827173.129539:0:27439:0:(rw26.c:933:ll_write_end()) page@00000000b22ff2ac[3 000000005ce0a915 2 1 00000000a23cbd2a 1 0 420989] 00000080:00200000:3.0:1787827173.129544:0:27439:0:(rw26.c:933:ll_write_end()) vmpage @0000000099ab2e04 fffffc0001001 3:0 ffff9e34c3d8f910 420989 lru 00000080:00200000:3.0:1787827173.129558:0:27439:0:(rw26.c:933:ll_write_end()) osc-page@00000000d77bf340 210045: 1< 0 - - > 2< 860344320 0 0 0x0 0x0 | 000000004db7037e 0000000052729811 > 3< 0 0 > 4< 0 0 8 484102144 - | - - + - > 5< - - + - | 0 - | 343 - -> 00000080:00200000:3.0:1787827173.129563:0:27439:0:(rw26.c:933:ll_write_end()) end page@00000000b22ff2ac 00000080:00200000:3.0:1787827173.129566:0:27439:0:(rw26.c:933:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1787827173.129570:0:27439:0:(vvp_io.c:1214:vvp_io_write_commit()) commit async pages: 1, from 0, to 1024 prio 0 00000008:00010000:3.0:1787827173.129583:0:27439:0:(osc_cache.c:565:osc_extent_merge()) ### extent: 000000001063fa34 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.129612:0:27439:0:(osc_cache.c:497:__osc_extent_remove()) ### extent: 000000001063fa34 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.129625:0:27439:0:(osc_cache.c:582:osc_extent_merge()) ### extent: 00000000df269729 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.129637:0:27439:0:(osc_cache.c:846:osc_extent_find()) ### extent: 00000000df269729 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.129657:0:27439:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff9e35d0e392f8, count 1, dirtied 1 00000080:00200000:3.0:1787827173.129662:0:27439:0:(vvp_io.c:1237:vvp_io_write_commit()) Committed 1 pages 1024 bytes, tot: 1024 00000080:00200000:3.0:1787827173.129668:0:27439:0:(vvp_io.c:1421:vvp_io_write_start()) oosfile: write bytes 0, result: 1024 00000020:00010000:3.0:1787827173.129678:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_release():245 00000020:00010000:3.0:1787827173.129685:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1787827173.129689:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c7d81d30 (ffff9e35da704cf8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1787827173.129697:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.129717:0:27439:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 17/0,0 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1787827173.129734:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_fini():72 00000020:00010000:3.0:1787827173.129738:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c7d81d30 (ffff9e35c614ce18/1) at cl_lock_fini():72 00000080:00200000:3.0:1787827173.129742:0:27439:0:(vvp_io.c:533:vvp_io_advance()) advancing 1024 bytes 00000080:00200000:3.0:1787827173.129746:0:27439:0:(file.c:2207:ll_file_io_generic()) Range unlock [420989, 420989] 00000080:00200000:3.0:1787827173.129751:0:27439:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787827173.129758:0:27439:0:(file.c:2240:ll_file_io_generic()) oosfile: 2 io complete with rc: 0, result: 1024, restart: 0 00000080:00200000:3.0:1787827173.129768:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724371968 i_blocks 3365288 00000080:00200000:3.0:1787827173.129774:0:27439:0:(file.c:2326:ll_file_io_generic()) iot: 2, result: 1024 00000080:00200008:3.0:1787827173.129787:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724371968, count: 1024 00000080:00200000:3.0:1787827173.129794:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 420989 of 1024 to 1024 bytes 00000080:00200000:3.0:1787827173.129820:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724371968, len 1024, copied 1024 00000080:00200000:3.0:1787827173.129831:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724372992 i_blocks 3365288 00000080:00200000:3.0:1787827173.129839:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.129847:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724372992, count: 1024 00000080:00200000:3.0:1787827173.129857:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 420989 of 2048 to 1024 bytes 00000080:00200000:3.0:1787827173.129863:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724372992, len 1024, copied 1024 00000080:00200000:3.0:1787827173.129871:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724374016 i_blocks 3365288 00000080:00200000:3.0:1787827173.129877:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.129884:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724374016, count: 1024 00000080:00200000:3.0:1787827173.129891:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 420989 of 3072 to 1024 bytes 00000080:00200000:3.0:1787827173.129898:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724374016, len 1024, copied 1024 00000080:00200000:3.0:1787827173.129905:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724375040 i_blocks 3365288 00000080:00200000:3.0:1787827173.129910:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.129930:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724375040, count: 1024 00000080:00200000:3.0:1787827173.129937:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 420990 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.129942:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 0, original count 1024 00000080:00200000:3.0:1787827173.129946:0:27439:0:(file.c:2090:ll_file_io_generic()) oosfile: write ppos: 1724375040, bytes: 1024 00000080:00200000:3.0:1787827173.129951:0:27439:0:(file.c:1848:ll_io_set_mirror()) oosfile: desiginated mirror: 0 00000020:00200000:3.0:1787827173.129959:0:27439:0:(cl_io.c:218:cl_io_rw_init()) header@000000007788600a[0x110, 368543, [0x200000402:0x27:0x0] hash] 00000020:00200000:3.0:1787827173.129962:0:27439:0:(cl_io.c:218:cl_io_rw_init()) io range: 2 [1724375040, 1724376064) 0 0 00000080:00200000:3.0:1787827173.129967:0:27439:0:(vvp_io.c:1889:vvp_io_init()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1787827173.129977:0:27439:0:(file.c:2170:ll_file_io_generic()) Range lock [420990, 420990] 00020000:00200000:3.0:1787827173.129984:0:27439:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 411 chunk: [1724375040, 1724376064) 1724376064, 1024 00020000:00200000:3.0:1787827173.129989:0:27439:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1787827173.129996:0:27439:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [420990, 420990] 00000008:00010000:3.0:1787827173.130005:0:27439:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1787827173.130010:0:27439:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff9e35c7d81d30, osc lock ffff9e35c2ddcc30, flags 0x0 00010000:00010000:3.0:1787827173.130023:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.130046:0:27439:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (860348416 860352511) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.130067:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 20/0,2 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.130084:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 20/0,2 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.130096:0:27439:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 19/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.130109:0:27439:0:(vvp_io.c:1339:vvp_io_write_start()) oosfile: write [1724375040, 1724376064) append 0 append lockpos 0 00000080:00200000:3.0:1787827173.130117:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 420990 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.130141:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724375040, len 1024, copied 1024 00000080:00200000:3.0:1787827173.130150:0:27439:0:(rw26.c:933:ll_write_end()) page@00000000ff1b847f[3 000000005ce0a915 2 1 00000000a23cbd2a 1 0 420990] 00000080:00200000:3.0:1787827173.130154:0:27439:0:(rw26.c:933:ll_write_end()) vmpage @000000004a77c1ff fffffc0001001 3:0 ffff9e34c3d8f0e8 420990 lru 00000080:00200000:3.0:1787827173.130171:0:27439:0:(rw26.c:933:ll_write_end()) osc-page@000000003aa34ec3 210046: 1< 0 - - > 2< 860348416 0 0 0x0 0x0 | 000000004db7037e 0000000052729811 > 3< 0 0 > 4< 0 0 8 484098048 - | - - + - > 5< - - + - | 0 - | 344 - -> 00000080:00200000:3.0:1787827173.130174:0:27439:0:(rw26.c:933:ll_write_end()) end page@00000000ff1b847f 00000080:00200000:3.0:1787827173.130177:0:27439:0:(rw26.c:933:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1787827173.130181:0:27439:0:(vvp_io.c:1214:vvp_io_write_commit()) commit async pages: 1, from 0, to 1024 prio 0 00000008:00010000:3.0:1787827173.130192:0:27439:0:(osc_cache.c:565:osc_extent_merge()) ### extent: 000000001063fa34 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.130214:0:27439:0:(osc_cache.c:497:__osc_extent_remove()) ### extent: 000000001063fa34 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.130222:0:27439:0:(osc_cache.c:582:osc_extent_merge()) ### extent: 00000000df269729 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.130243:0:27439:0:(osc_cache.c:846:osc_extent_find()) ### extent: 00000000df269729 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.130264:0:27439:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff9e35d0e392f8, count 1, dirtied 1 00000080:00200000:3.0:1787827173.130270:0:27439:0:(vvp_io.c:1237:vvp_io_write_commit()) Committed 1 pages 1024 bytes, tot: 1024 00000080:00200000:3.0:1787827173.130275:0:27439:0:(vvp_io.c:1421:vvp_io_write_start()) oosfile: write bytes 0, result: 1024 00000020:00010000:3.0:1787827173.130286:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_release():245 00000020:00010000:3.0:1787827173.130295:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1787827173.130300:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c7d81d30 (ffff9e35da704cf8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1787827173.130305:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.130321:0:27439:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 17/0,0 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1787827173.130337:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_fini():72 00000020:00010000:3.0:1787827173.130344:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c7d81d30 (ffff9e35c614ce18/1) at cl_lock_fini():72 00000080:00200000:3.0:1787827173.130350:0:27439:0:(vvp_io.c:533:vvp_io_advance()) advancing 1024 bytes 00000080:00200000:3.0:1787827173.130356:0:27439:0:(file.c:2207:ll_file_io_generic()) Range unlock [420990, 420990] 00000080:00200000:3.0:1787827173.130362:0:27439:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787827173.130368:0:27439:0:(file.c:2240:ll_file_io_generic()) oosfile: 2 io complete with rc: 0, result: 1024, restart: 0 00000080:00200000:3.0:1787827173.130376:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724376064 i_blocks 3365288 00000080:00200000:3.0:1787827173.130381:0:27439:0:(file.c:2326:ll_file_io_generic()) iot: 2, result: 1024 00000080:00200008:3.0:1787827173.130392:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724376064, count: 1024 00000080:00200000:3.0:1787827173.130399:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 420990 of 1024 to 1024 bytes 00000080:00200000:3.0:1787827173.130405:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724376064, len 1024, copied 1024 00000080:00200000:3.0:1787827173.130418:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724377088 i_blocks 3365288 00000080:00200000:3.0:1787827173.130426:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.130436:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724377088, count: 1024 00000080:00200000:3.0:1787827173.130445:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 420990 of 2048 to 1024 bytes 00000080:00200000:3.0:1787827173.130450:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724377088, len 1024, copied 1024 00000080:00200000:3.0:1787827173.130459:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724378112 i_blocks 3365288 00000080:00200000:3.0:1787827173.130465:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.130476:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724378112, count: 1024 00000080:00200000:3.0:1787827173.130485:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 420990 of 3072 to 1024 bytes 00000080:00200000:3.0:1787827173.130492:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724378112, len 1024, copied 1024 00000080:00200000:3.0:1787827173.130503:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724379136 i_blocks 3365288 00000080:00200000:3.0:1787827173.130510:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.130519:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724379136, count: 1024 00000080:00200000:3.0:1787827173.130526:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 420991 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.130531:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 0, original count 1024 00000080:00200000:3.0:1787827173.130536:0:27439:0:(file.c:2090:ll_file_io_generic()) oosfile: write ppos: 1724379136, bytes: 1024 00000080:00200000:3.0:1787827173.130543:0:27439:0:(file.c:1848:ll_io_set_mirror()) oosfile: desiginated mirror: 0 00000020:00200000:3.0:1787827173.130553:0:27439:0:(cl_io.c:218:cl_io_rw_init()) header@000000007788600a[0x110, 368544, [0x200000402:0x27:0x0] hash] 00000020:00200000:3.0:1787827173.130556:0:27439:0:(cl_io.c:218:cl_io_rw_init()) io range: 2 [1724379136, 1724380160) 0 0 00000080:00200000:3.0:1787827173.130561:0:27439:0:(vvp_io.c:1889:vvp_io_init()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1787827173.130573:0:27439:0:(file.c:2170:ll_file_io_generic()) Range lock [420991, 420991] 00020000:00200000:3.0:1787827173.130580:0:27439:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 411 chunk: [1724379136, 1724380160) 1724380160, 1024 00020000:00200000:3.0:1787827173.130587:0:27439:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1787827173.130595:0:27439:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [420991, 420991] 00000008:00010000:3.0:1787827173.130605:0:27439:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1787827173.130608:0:27439:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff9e35c7d81d30, osc lock ffff9e35c2ddcc30, flags 0x0 00010000:00010000:3.0:1787827173.130620:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.130640:0:27439:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (860352512 860356607) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.130661:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 20/0,2 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.130685:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 20/0,2 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.130698:0:27439:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 19/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.130711:0:27439:0:(vvp_io.c:1339:vvp_io_write_start()) oosfile: write [1724379136, 1724380160) append 0 append lockpos 0 00000080:00200000:3.0:1787827173.130722:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 420991 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.130747:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724379136, len 1024, copied 1024 00000080:00200000:3.0:1787827173.130758:0:27439:0:(rw26.c:933:ll_write_end()) page@00000000b7f781ad[3 000000005ce0a915 2 1 00000000a23cbd2a 1 0 420991] 00000080:00200000:3.0:1787827173.130765:0:27439:0:(rw26.c:933:ll_write_end()) vmpage @00000000cf931011 fffffc0001001 3:0 ffff9e34c3d8fcb0 420991 lru 00000080:00200000:3.0:1787827173.130777:0:27439:0:(rw26.c:933:ll_write_end()) osc-page@0000000036e610bd 210047: 1< 0 - - > 2< 860352512 0 0 0x0 0x0 | 000000004db7037e 0000000052729811 > 3< 0 0 > 4< 0 0 8 484093952 - | - - + - > 5< - - + - | 0 - | 345 - -> 00000080:00200000:3.0:1787827173.130781:0:27439:0:(rw26.c:933:ll_write_end()) end page@00000000b7f781ad 00000080:00200000:3.0:1787827173.130784:0:27439:0:(rw26.c:933:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1787827173.130789:0:27439:0:(vvp_io.c:1214:vvp_io_write_commit()) commit async pages: 1, from 0, to 1024 prio 0 00000008:00010000:3.0:1787827173.130823:0:27439:0:(osc_cache.c:565:osc_extent_merge()) ### extent: 000000001063fa34 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.130858:0:27439:0:(osc_cache.c:497:__osc_extent_remove()) ### extent: 000000001063fa34 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.130877:0:27439:0:(osc_cache.c:582:osc_extent_merge()) ### extent: 00000000df269729 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.130897:0:27439:0:(osc_cache.c:846:osc_extent_find()) ### extent: 00000000df269729 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.130915:0:27439:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff9e35d0e392f8, count 1, dirtied 1 00000080:00200000:3.0:1787827173.130934:0:27439:0:(vvp_io.c:1237:vvp_io_write_commit()) Committed 1 pages 1024 bytes, tot: 1024 00000080:00200000:3.0:1787827173.130939:0:27439:0:(vvp_io.c:1421:vvp_io_write_start()) oosfile: write bytes 0, result: 1024 00000020:00010000:3.0:1787827173.130949:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_release():245 00000020:00010000:3.0:1787827173.130956:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1787827173.130964:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c7d81d30 (ffff9e35da704cf8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1787827173.130972:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.130994:0:27439:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 17/0,0 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1787827173.131019:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_fini():72 00000020:00010000:3.0:1787827173.131026:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c7d81d30 (ffff9e35c614ce18/1) at cl_lock_fini():72 00000080:00200000:3.0:1787827173.131032:0:27439:0:(vvp_io.c:533:vvp_io_advance()) advancing 1024 bytes 00000080:00200000:3.0:1787827173.131038:0:27439:0:(file.c:2207:ll_file_io_generic()) Range unlock [420991, 420991] 00000080:00200000:3.0:1787827173.131046:0:27439:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787827173.131049:0:27439:0:(file.c:2240:ll_file_io_generic()) oosfile: 2 io complete with rc: 0, result: 1024, restart: 0 00000080:00200000:3.0:1787827173.131055:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724380160 i_blocks 3365288 00000080:00200000:3.0:1787827173.131059:0:27439:0:(file.c:2326:ll_file_io_generic()) iot: 2, result: 1024 00000080:00200008:3.0:1787827173.131067:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724380160, count: 1024 00000080:00200000:3.0:1787827173.131075:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 420991 of 1024 to 1024 bytes 00000080:00200000:3.0:1787827173.131083:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724380160, len 1024, copied 1024 00000080:00200000:3.0:1787827173.131092:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724381184 i_blocks 3365288 00000080:00200000:3.0:1787827173.131097:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.131103:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724381184, count: 1024 00000080:00200000:3.0:1787827173.131110:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 420991 of 2048 to 1024 bytes 00000080:00200000:3.0:1787827173.131116:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724381184, len 1024, copied 1024 00000080:00200000:3.0:1787827173.131123:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724382208 i_blocks 3365288 00000080:00200000:3.0:1787827173.131128:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.131138:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724382208, count: 1024 00000080:00200000:3.0:1787827173.131145:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 420991 of 3072 to 1024 bytes 00000080:00200000:3.0:1787827173.131150:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724382208, len 1024, copied 1024 00000080:00200000:3.0:1787827173.131158:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724383232 i_blocks 3365288 00000080:00200000:3.0:1787827173.131166:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.131174:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724383232, count: 1024 00000080:00200000:3.0:1787827173.131183:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 420992 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.131188:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 0, original count 1024 00000080:00200000:3.0:1787827173.131194:0:27439:0:(file.c:2090:ll_file_io_generic()) oosfile: write ppos: 1724383232, bytes: 1024 00000080:00200000:3.0:1787827173.131201:0:27439:0:(file.c:1848:ll_io_set_mirror()) oosfile: desiginated mirror: 0 00000020:00200000:3.0:1787827173.131210:0:27439:0:(cl_io.c:218:cl_io_rw_init()) header@000000007788600a[0x110, 368545, [0x200000402:0x27:0x0] hash] 00000020:00200000:3.0:1787827173.131215:0:27439:0:(cl_io.c:218:cl_io_rw_init()) io range: 2 [1724383232, 1724384256) 0 0 00000080:00200000:3.0:1787827173.131220:0:27439:0:(vvp_io.c:1889:vvp_io_init()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1787827173.131230:0:27439:0:(file.c:2170:ll_file_io_generic()) Range lock [420992, 420992] 00020000:00200000:3.0:1787827173.131237:0:27439:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 411 chunk: [1724383232, 1724384256) 1724384256, 1024 00020000:00200000:3.0:1787827173.131244:0:27439:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1787827173.131252:0:27439:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [420992, 420992] 00000008:00010000:3.0:1787827173.131261:0:27439:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1787827173.131265:0:27439:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff9e35c7d81d30, osc lock ffff9e35c2ddcc30, flags 0x0 00010000:00010000:3.0:1787827173.131278:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.131295:0:27439:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (860356608 860360703) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.131313:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 20/0,2 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.131329:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 20/0,2 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.131342:0:27439:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 19/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.131359:0:27439:0:(vvp_io.c:1339:vvp_io_write_start()) oosfile: write [1724383232, 1724384256) append 0 append lockpos 0 00000080:00200000:3.0:1787827173.131369:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 420992 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.131394:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724383232, len 1024, copied 1024 00000080:00200000:3.0:1787827173.131405:0:27439:0:(rw26.c:933:ll_write_end()) page@000000002f266ac9[3 000000005ce0a915 2 1 00000000a23cbd2a 1 0 420992] 00000080:00200000:3.0:1787827173.131411:0:27439:0:(rw26.c:933:ll_write_end()) vmpage @00000000580b5f0f fffffc0001001 3:0 ffff9e34c3d8f1d0 420992 lru 00000080:00200000:3.0:1787827173.131425:0:27439:0:(rw26.c:933:ll_write_end()) osc-page@000000000330d56b 210048: 1< 0 - - > 2< 860356608 0 0 0x0 0x0 | 000000004db7037e 0000000052729811 > 3< 0 0 > 4< 0 0 8 484089856 - | - - + - > 5< - - + - | 0 - | 346 - -> 00000080:00200000:3.0:1787827173.131430:0:27439:0:(rw26.c:933:ll_write_end()) end page@000000002f266ac9 00000080:00200000:3.0:1787827173.131434:0:27439:0:(rw26.c:933:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1787827173.131437:0:27439:0:(vvp_io.c:1214:vvp_io_write_commit()) commit async pages: 1, from 0, to 1024 prio 0 00000008:00010000:3.0:1787827173.131451:0:27439:0:(osc_cache.c:565:osc_extent_merge()) ### extent: 000000001063fa34 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.131480:0:27439:0:(osc_cache.c:497:__osc_extent_remove()) ### extent: 000000001063fa34 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.131498:0:27439:0:(osc_cache.c:582:osc_extent_merge()) ### extent: 00000000df269729 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.131520:0:27439:0:(osc_cache.c:846:osc_extent_find()) ### extent: 00000000df269729 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.131548:0:27439:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff9e35d0e392f8, count 1, dirtied 1 00000080:00200000:3.0:1787827173.131553:0:27439:0:(vvp_io.c:1237:vvp_io_write_commit()) Committed 1 pages 1024 bytes, tot: 1024 00000080:00200000:3.0:1787827173.131558:0:27439:0:(vvp_io.c:1421:vvp_io_write_start()) oosfile: write bytes 0, result: 1024 00000020:00010000:3.0:1787827173.131568:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_release():245 00000020:00010000:3.0:1787827173.131572:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1787827173.131575:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c7d81d30 (ffff9e35da704cf8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1787827173.131583:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.131598:0:27439:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 17/0,0 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1787827173.131609:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_fini():72 00000020:00010000:3.0:1787827173.131617:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c7d81d30 (ffff9e35c614ce18/1) at cl_lock_fini():72 00000080:00200000:3.0:1787827173.131626:0:27439:0:(vvp_io.c:533:vvp_io_advance()) advancing 1024 bytes 00000080:00200000:3.0:1787827173.131631:0:27439:0:(file.c:2207:ll_file_io_generic()) Range unlock [420992, 420992] 00000080:00200000:3.0:1787827173.131637:0:27439:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787827173.131645:0:27439:0:(file.c:2240:ll_file_io_generic()) oosfile: 2 io complete with rc: 0, result: 1024, restart: 0 00000080:00200000:3.0:1787827173.131654:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724384256 i_blocks 3365288 00000080:00200000:3.0:1787827173.131658:0:27439:0:(file.c:2326:ll_file_io_generic()) iot: 2, result: 1024 00000080:00200008:3.0:1787827173.131668:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724384256, count: 1024 00000080:00200000:3.0:1787827173.131676:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 420992 of 1024 to 1024 bytes 00000080:00200000:3.0:1787827173.131682:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724384256, len 1024, copied 1024 00000080:00200000:3.0:1787827173.131688:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724385280 i_blocks 3365288 00000080:00200000:3.0:1787827173.131691:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.131695:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724385280, count: 1024 00000080:00200000:3.0:1787827173.131698:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 420992 of 2048 to 1024 bytes 00000080:00200000:3.0:1787827173.131701:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724385280, len 1024, copied 1024 00000080:00200000:3.0:1787827173.131708:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724386304 i_blocks 3365288 00000080:00200000:3.0:1787827173.131713:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.131719:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724386304, count: 1024 00000080:00200000:3.0:1787827173.131724:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 420992 of 3072 to 1024 bytes 00000080:00200000:3.0:1787827173.131726:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724386304, len 1024, copied 1024 00000080:00200000:3.0:1787827173.131731:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724387328 i_blocks 3365288 00000080:00200000:3.0:1787827173.131734:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.131738:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724387328, count: 1024 00000080:00200000:3.0:1787827173.131741:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 420993 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.131744:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 0, original count 1024 00000080:00200000:3.0:1787827173.131746:0:27439:0:(file.c:2090:ll_file_io_generic()) oosfile: write ppos: 1724387328, bytes: 1024 00000080:00200000:3.0:1787827173.131748:0:27439:0:(file.c:1848:ll_io_set_mirror()) oosfile: desiginated mirror: 0 00000020:00200000:3.0:1787827173.131753:0:27439:0:(cl_io.c:218:cl_io_rw_init()) header@000000007788600a[0x110, 368546, [0x200000402:0x27:0x0] hash] 00000020:00200000:3.0:1787827173.131754:0:27439:0:(cl_io.c:218:cl_io_rw_init()) io range: 2 [1724387328, 1724388352) 0 0 00000080:00200000:3.0:1787827173.131757:0:27439:0:(vvp_io.c:1889:vvp_io_init()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1787827173.131764:0:27439:0:(file.c:2170:ll_file_io_generic()) Range lock [420993, 420993] 00020000:00200000:3.0:1787827173.131767:0:27439:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 411 chunk: [1724387328, 1724388352) 1724388352, 1024 00020000:00200000:3.0:1787827173.131770:0:27439:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1787827173.131776:0:27439:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [420993, 420993] 00000008:00010000:3.0:1787827173.131782:0:27439:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1787827173.131785:0:27439:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff9e35c7d81d30, osc lock ffff9e35c2ddcc30, flags 0x0 00010000:00010000:3.0:1787827173.131794:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.131824:0:27439:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (860360704 860364799) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.131833:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 20/0,2 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.131841:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 20/0,2 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.131848:0:27439:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 19/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.131855:0:27439:0:(vvp_io.c:1339:vvp_io_write_start()) oosfile: write [1724387328, 1724388352) append 0 append lockpos 0 00000080:00200000:3.0:1787827173.131860:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 420993 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.131878:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724387328, len 1024, copied 1024 00000080:00200000:3.0:1787827173.131883:0:27439:0:(rw26.c:933:ll_write_end()) page@0000000030b3a299[3 000000005ce0a915 2 1 00000000a23cbd2a 1 0 420993] 00000080:00200000:3.0:1787827173.131886:0:27439:0:(rw26.c:933:ll_write_end()) vmpage @00000000f657b4da fffffc0001001 3:0 ffff9e34c3d8f658 420993 lru 00000080:00200000:3.0:1787827173.131892:0:27439:0:(rw26.c:933:ll_write_end()) osc-page@00000000e3925cfd 210049: 1< 0 - - > 2< 860360704 0 0 0x0 0x0 | 000000004db7037e 0000000052729811 > 3< 0 0 > 4< 0 0 8 484085760 - | - - + - > 5< - - + - | 0 - | 347 - -> 00000080:00200000:3.0:1787827173.131894:0:27439:0:(rw26.c:933:ll_write_end()) end page@0000000030b3a299 00000080:00200000:3.0:1787827173.131896:0:27439:0:(rw26.c:933:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1787827173.131898:0:27439:0:(vvp_io.c:1214:vvp_io_write_commit()) commit async pages: 1, from 0, to 1024 prio 0 00000008:00010000:3.0:1787827173.131907:0:27439:0:(osc_cache.c:565:osc_extent_merge()) ### extent: 000000001063fa34 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.131933:0:27439:0:(osc_cache.c:497:__osc_extent_remove()) ### extent: 000000001063fa34 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.131940:0:27439:0:(osc_cache.c:582:osc_extent_merge()) ### extent: 00000000df269729 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.131948:0:27439:0:(osc_cache.c:846:osc_extent_find()) ### extent: 00000000df269729 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.131962:0:27439:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff9e35d0e392f8, count 1, dirtied 1 00000080:00200000:3.0:1787827173.131966:0:27439:0:(vvp_io.c:1237:vvp_io_write_commit()) Committed 1 pages 1024 bytes, tot: 1024 00000080:00200000:3.0:1787827173.131968:0:27439:0:(vvp_io.c:1421:vvp_io_write_start()) oosfile: write bytes 0, result: 1024 00000020:00010000:3.0:1787827173.131976:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_release():245 00000020:00010000:3.0:1787827173.131978:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1787827173.131981:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c7d81d30 (ffff9e35da704cf8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1787827173.131985:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.131994:0:27439:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 17/0,0 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1787827173.132003:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_fini():72 00000020:00010000:3.0:1787827173.132006:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c7d81d30 (ffff9e35c614ce18/1) at cl_lock_fini():72 00000080:00200000:3.0:1787827173.132011:0:27439:0:(vvp_io.c:533:vvp_io_advance()) advancing 1024 bytes 00000080:00200000:3.0:1787827173.132015:0:27439:0:(file.c:2207:ll_file_io_generic()) Range unlock [420993, 420993] 00000080:00200000:3.0:1787827173.132019:0:27439:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787827173.132022:0:27439:0:(file.c:2240:ll_file_io_generic()) oosfile: 2 io complete with rc: 0, result: 1024, restart: 0 00000080:00200000:3.0:1787827173.132028:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724388352 i_blocks 3365288 00000080:00200000:3.0:1787827173.132030:0:27439:0:(file.c:2326:ll_file_io_generic()) iot: 2, result: 1024 00000080:00200008:3.0:1787827173.132038:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724388352, count: 1024 00000080:00200000:3.0:1787827173.132042:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 420993 of 1024 to 1024 bytes 00000080:00200000:3.0:1787827173.132046:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724388352, len 1024, copied 1024 00000080:00200000:3.0:1787827173.132051:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724389376 i_blocks 3365288 00000080:00200000:3.0:1787827173.132054:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.132058:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724389376, count: 1024 00000080:00200000:3.0:1787827173.132061:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 420993 of 2048 to 1024 bytes 00000080:00200000:3.0:1787827173.132064:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724389376, len 1024, copied 1024 00000080:00200000:3.0:1787827173.132069:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724390400 i_blocks 3365288 00000080:00200000:3.0:1787827173.132072:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.132076:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724390400, count: 1024 00000080:00200000:3.0:1787827173.132080:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 420993 of 3072 to 1024 bytes 00000080:00200000:3.0:1787827173.132082:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724390400, len 1024, copied 1024 00000080:00200000:3.0:1787827173.132088:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724391424 i_blocks 3365288 00000080:00200000:3.0:1787827173.132091:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.132096:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724391424, count: 1024 00000080:00200000:3.0:1787827173.132099:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 420994 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.132102:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 0, original count 1024 00000080:00200000:3.0:1787827173.132104:0:27439:0:(file.c:2090:ll_file_io_generic()) oosfile: write ppos: 1724391424, bytes: 1024 00000080:00200000:3.0:1787827173.132107:0:27439:0:(file.c:1848:ll_io_set_mirror()) oosfile: desiginated mirror: 0 00000020:00200000:3.0:1787827173.132111:0:27439:0:(cl_io.c:218:cl_io_rw_init()) header@000000007788600a[0x110, 368547, [0x200000402:0x27:0x0] hash] 00000020:00200000:3.0:1787827173.132113:0:27439:0:(cl_io.c:218:cl_io_rw_init()) io range: 2 [1724391424, 1724392448) 0 0 00000080:00200000:3.0:1787827173.132115:0:27439:0:(vvp_io.c:1889:vvp_io_init()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1787827173.132121:0:27439:0:(file.c:2170:ll_file_io_generic()) Range lock [420994, 420994] 00020000:00200000:3.0:1787827173.132125:0:27439:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 411 chunk: [1724391424, 1724392448) 1724392448, 1024 00020000:00200000:3.0:1787827173.132127:0:27439:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1787827173.132133:0:27439:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [420994, 420994] 00000008:00010000:3.0:1787827173.132139:0:27439:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1787827173.132142:0:27439:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff9e35c7d81d30, osc lock ffff9e35c2ddcc30, flags 0x0 00010000:00010000:3.0:1787827173.132151:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.132160:0:27439:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (860364800 860368895) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.132169:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 20/0,2 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.132177:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 20/0,2 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.132184:0:27439:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 19/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.132191:0:27439:0:(vvp_io.c:1339:vvp_io_write_start()) oosfile: write [1724391424, 1724392448) append 0 append lockpos 0 00000080:00200000:3.0:1787827173.132195:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 420994 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.132214:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724391424, len 1024, copied 1024 00000080:00200000:3.0:1787827173.132219:0:27439:0:(rw26.c:933:ll_write_end()) page@00000000aa0b6946[3 000000005ce0a915 2 1 00000000a23cbd2a 1 0 420994] 00000080:00200000:3.0:1787827173.132222:0:27439:0:(rw26.c:933:ll_write_end()) vmpage @000000007bcdb13b fffffc0001001 3:0 ffff9e34c3d8f3a0 420994 lru 00000080:00200000:3.0:1787827173.132227:0:27439:0:(rw26.c:933:ll_write_end()) osc-page@00000000dbdf3652 210050: 1< 0 - - > 2< 860364800 0 0 0x0 0x0 | 000000004db7037e 0000000052729811 > 3< 0 0 > 4< 0 0 8 484081664 - | - - + - > 5< - - + - | 0 - | 348 - -> 00000080:00200000:3.0:1787827173.132230:0:27439:0:(rw26.c:933:ll_write_end()) end page@00000000aa0b6946 00000080:00200000:3.0:1787827173.132231:0:27439:0:(rw26.c:933:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1787827173.132233:0:27439:0:(vvp_io.c:1214:vvp_io_write_commit()) commit async pages: 1, from 0, to 1024 prio 0 00000008:00010000:3.0:1787827173.132243:0:27439:0:(osc_cache.c:565:osc_extent_merge()) ### extent: 000000001063fa34 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.132256:0:27439:0:(osc_cache.c:497:__osc_extent_remove()) ### extent: 000000001063fa34 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.132263:0:27439:0:(osc_cache.c:582:osc_extent_merge()) ### extent: 00000000df269729 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.132272:0:27439:0:(osc_cache.c:846:osc_extent_find()) ### extent: 00000000df269729 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.132286:0:27439:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff9e35d0e392f8, count 1, dirtied 1 00000080:00200000:3.0:1787827173.132288:0:27439:0:(vvp_io.c:1237:vvp_io_write_commit()) Committed 1 pages 1024 bytes, tot: 1024 00000080:00200000:3.0:1787827173.132290:0:27439:0:(vvp_io.c:1421:vvp_io_write_start()) oosfile: write bytes 0, result: 1024 00000020:00010000:3.0:1787827173.132296:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_release():245 00000020:00010000:3.0:1787827173.132298:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1787827173.132300:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c7d81d30 (ffff9e35da704cf8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1787827173.132303:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.132309:0:27439:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 17/0,0 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1787827173.132316:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_fini():72 00000020:00010000:3.0:1787827173.132317:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c7d81d30 (ffff9e35c614ce18/1) at cl_lock_fini():72 00000080:00200000:3.0:1787827173.132321:0:27439:0:(vvp_io.c:533:vvp_io_advance()) advancing 1024 bytes 00000080:00200000:3.0:1787827173.132324:0:27439:0:(file.c:2207:ll_file_io_generic()) Range unlock [420994, 420994] 00000080:00200000:3.0:1787827173.132328:0:27439:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787827173.132330:0:27439:0:(file.c:2240:ll_file_io_generic()) oosfile: 2 io complete with rc: 0, result: 1024, restart: 0 00000080:00200000:3.0:1787827173.132334:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724392448 i_blocks 3365288 00000080:00200000:3.0:1787827173.132336:0:27439:0:(file.c:2326:ll_file_io_generic()) iot: 2, result: 1024 00000080:00200008:3.0:1787827173.132342:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724392448, count: 1024 00000080:00200000:3.0:1787827173.132346:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 420994 of 1024 to 1024 bytes 00000080:00200000:3.0:1787827173.132349:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724392448, len 1024, copied 1024 00000080:00200000:3.0:1787827173.132354:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724393472 i_blocks 3365288 00000080:00200000:3.0:1787827173.132357:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.132362:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724393472, count: 1024 00000080:00200000:3.0:1787827173.132365:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 420994 of 2048 to 1024 bytes 00000080:00200000:3.0:1787827173.132368:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724393472, len 1024, copied 1024 00000080:00200000:3.0:1787827173.132374:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724394496 i_blocks 3365288 00000080:00200000:3.0:1787827173.132376:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.132381:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724394496, count: 1024 00000080:00200000:3.0:1787827173.132385:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 420994 of 3072 to 1024 bytes 00000080:00200000:3.0:1787827173.132388:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724394496, len 1024, copied 1024 00000080:00200000:3.0:1787827173.132393:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724395520 i_blocks 3365288 00000080:00200000:3.0:1787827173.132396:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.132401:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724395520, count: 1024 00000080:00200000:3.0:1787827173.132405:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 420995 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.132407:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 0, original count 1024 00000080:00200000:3.0:1787827173.132409:0:27439:0:(file.c:2090:ll_file_io_generic()) oosfile: write ppos: 1724395520, bytes: 1024 00000080:00200000:3.0:1787827173.132411:0:27439:0:(file.c:1848:ll_io_set_mirror()) oosfile: desiginated mirror: 0 00000020:00200000:3.0:1787827173.132413:0:27439:0:(cl_io.c:218:cl_io_rw_init()) header@000000007788600a[0x110, 368548, [0x200000402:0x27:0x0] hash] 00000020:00200000:3.0:1787827173.132415:0:27439:0:(cl_io.c:218:cl_io_rw_init()) io range: 2 [1724395520, 1724396544) 0 0 00000080:00200000:3.0:1787827173.132417:0:27439:0:(vvp_io.c:1889:vvp_io_init()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1787827173.132421:0:27439:0:(file.c:2170:ll_file_io_generic()) Range lock [420995, 420995] 00020000:00200000:3.0:1787827173.132424:0:27439:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 411 chunk: [1724395520, 1724396544) 1724396544, 1024 00020000:00200000:3.0:1787827173.132426:0:27439:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1787827173.132431:0:27439:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [420995, 420995] 00000008:00010000:3.0:1787827173.132435:0:27439:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1787827173.132437:0:27439:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff9e35c7d81d30, osc lock ffff9e35c2ddcc30, flags 0x0 00010000:00010000:3.0:1787827173.132444:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.132450:0:27439:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (860368896 860372991) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.132457:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 20/0,2 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.132462:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 20/0,2 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.132467:0:27439:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 19/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.132471:0:27439:0:(vvp_io.c:1339:vvp_io_write_start()) oosfile: write [1724395520, 1724396544) append 0 append lockpos 0 00000080:00200000:3.0:1787827173.132475:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 420995 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.132489:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724395520, len 1024, copied 1024 00000080:00200000:3.0:1787827173.132493:0:27439:0:(rw26.c:933:ll_write_end()) page@00000000f1630bb3[3 000000005ce0a915 2 1 00000000a23cbd2a 1 0 420995] 00000080:00200000:3.0:1787827173.132495:0:27439:0:(rw26.c:933:ll_write_end()) vmpage @00000000a600d882 fffffc0001001 3:0 ffff9e34c3d8fd98 420995 lru 00000080:00200000:3.0:1787827173.132498:0:27439:0:(rw26.c:933:ll_write_end()) osc-page@00000000d1a13592 210051: 1< 0 - - > 2< 860368896 0 0 0x0 0x0 | 000000004db7037e 0000000052729811 > 3< 0 0 > 4< 0 0 8 484077568 - | - - + - > 5< - - + - | 0 - | 349 - -> 00000080:00200000:3.0:1787827173.132500:0:27439:0:(rw26.c:933:ll_write_end()) end page@00000000f1630bb3 00000080:00200000:3.0:1787827173.132501:0:27439:0:(rw26.c:933:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1787827173.132502:0:27439:0:(vvp_io.c:1214:vvp_io_write_commit()) commit async pages: 1, from 0, to 1024 prio 0 00000008:00010000:3.0:1787827173.132508:0:27439:0:(osc_cache.c:565:osc_extent_merge()) ### extent: 000000001063fa34 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.132517:0:27439:0:(osc_cache.c:497:__osc_extent_remove()) ### extent: 000000001063fa34 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.132521:0:27439:0:(osc_cache.c:582:osc_extent_merge()) ### extent: 00000000df269729 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.132526:0:27439:0:(osc_cache.c:846:osc_extent_find()) ### extent: 00000000df269729 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.132533:0:27439:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff9e35d0e392f8, count 1, dirtied 1 00000080:00200000:3.0:1787827173.132535:0:27439:0:(vvp_io.c:1237:vvp_io_write_commit()) Committed 1 pages 1024 bytes, tot: 1024 00000080:00200000:3.0:1787827173.132536:0:27439:0:(vvp_io.c:1421:vvp_io_write_start()) oosfile: write bytes 0, result: 1024 00000020:00010000:3.0:1787827173.132540:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_release():245 00000020:00010000:3.0:1787827173.132541:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1787827173.132542:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c7d81d30 (ffff9e35da704cf8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1787827173.132545:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.132550:0:27439:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 17/0,0 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1787827173.132557:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_fini():72 00000020:00010000:3.0:1787827173.132563:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c7d81d30 (ffff9e35c614ce18/1) at cl_lock_fini():72 00000080:00200000:3.0:1787827173.132567:0:27439:0:(vvp_io.c:533:vvp_io_advance()) advancing 1024 bytes 00000080:00200000:3.0:1787827173.132569:0:27439:0:(file.c:2207:ll_file_io_generic()) Range unlock [420995, 420995] 00000080:00200000:3.0:1787827173.132572:0:27439:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787827173.132574:0:27439:0:(file.c:2240:ll_file_io_generic()) oosfile: 2 io complete with rc: 0, result: 1024, restart: 0 00000080:00200000:3.0:1787827173.132577:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724396544 i_blocks 3365288 00000080:00200000:3.0:1787827173.132579:0:27439:0:(file.c:2326:ll_file_io_generic()) iot: 2, result: 1024 00000080:00200008:3.0:1787827173.132584:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724396544, count: 1024 00000080:00200000:3.0:1787827173.132587:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 420995 of 1024 to 1024 bytes 00000080:00200000:3.0:1787827173.132590:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724396544, len 1024, copied 1024 00000080:00200000:3.0:1787827173.132592:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724397568 i_blocks 3365288 00000080:00200000:3.0:1787827173.132594:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.132596:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724397568, count: 1024 00000080:00200000:3.0:1787827173.132598:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 420995 of 2048 to 1024 bytes 00000080:00200000:3.0:1787827173.132600:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724397568, len 1024, copied 1024 00000080:00200000:3.0:1787827173.132602:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724398592 i_blocks 3365288 00000080:00200000:3.0:1787827173.132604:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.132606:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724398592, count: 1024 00000080:00200000:3.0:1787827173.132608:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 420995 of 3072 to 1024 bytes 00000080:00200000:3.0:1787827173.132609:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724398592, len 1024, copied 1024 00000080:00200000:3.0:1787827173.132612:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724399616 i_blocks 3365288 00000080:00200000:3.0:1787827173.132613:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.132615:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724399616, count: 1024 00000080:00200000:3.0:1787827173.132617:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 420996 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.132618:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 0, original count 1024 00000080:00200000:3.0:1787827173.132619:0:27439:0:(file.c:2090:ll_file_io_generic()) oosfile: write ppos: 1724399616, bytes: 1024 00000080:00200000:3.0:1787827173.132620:0:27439:0:(file.c:1848:ll_io_set_mirror()) oosfile: desiginated mirror: 0 00000020:00200000:3.0:1787827173.132623:0:27439:0:(cl_io.c:218:cl_io_rw_init()) header@000000007788600a[0x110, 368549, [0x200000402:0x27:0x0] hash] 00000020:00200000:3.0:1787827173.132623:0:27439:0:(cl_io.c:218:cl_io_rw_init()) io range: 2 [1724399616, 1724400640) 0 0 00000080:00200000:3.0:1787827173.132625:0:27439:0:(vvp_io.c:1889:vvp_io_init()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1787827173.132628:0:27439:0:(file.c:2170:ll_file_io_generic()) Range lock [420996, 420996] 00020000:00200000:3.0:1787827173.132630:0:27439:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 411 chunk: [1724399616, 1724400640) 1724400640, 1024 00020000:00200000:3.0:1787827173.132631:0:27439:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1787827173.132634:0:27439:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [420996, 420996] 00000008:00010000:3.0:1787827173.132638:0:27439:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1787827173.132640:0:27439:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff9e35c7d81d30, osc lock ffff9e35c2ddcc30, flags 0x0 00010000:00010000:3.0:1787827173.132645:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.132651:0:27439:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (860372992 860377087) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.132657:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 20/0,2 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.132661:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 20/0,2 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.132665:0:27439:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 19/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.132670:0:27439:0:(vvp_io.c:1339:vvp_io_write_start()) oosfile: write [1724399616, 1724400640) append 0 append lockpos 0 00000080:00200000:3.0:1787827173.132674:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 420996 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.132696:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724399616, len 1024, copied 1024 00000080:00200000:3.0:1787827173.132699:0:27439:0:(rw26.c:933:ll_write_end()) page@000000009fdf1429[3 000000005ce0a915 2 1 00000000a23cbd2a 1 0 420996] 00000080:00200000:3.0:1787827173.132701:0:27439:0:(rw26.c:933:ll_write_end()) vmpage @000000004b41a405 fffffc0001021 2:0 ffff9e34c3d8f000 420996 lru 00000080:00200000:3.0:1787827173.132705:0:27439:0:(rw26.c:933:ll_write_end()) osc-page@00000000854b8f3c 210052: 1< 0 - - > 2< 860372992 0 0 0x0 0x0 | 000000004db7037e 0000000052729811 > 3< 0 0 > 4< 0 0 8 484073472 - | - - + - > 5< - - + - | 0 - | 350 - -> 00000080:00200000:3.0:1787827173.132706:0:27439:0:(rw26.c:933:ll_write_end()) end page@000000009fdf1429 00000080:00200000:3.0:1787827173.132707:0:27439:0:(rw26.c:933:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1787827173.132709:0:27439:0:(vvp_io.c:1214:vvp_io_write_commit()) commit async pages: 1, from 0, to 1024 prio 0 00000008:00010000:3.0:1787827173.132714:0:27439:0:(osc_cache.c:565:osc_extent_merge()) ### extent: 000000001063fa34 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.132723:0:27439:0:(osc_cache.c:497:__osc_extent_remove()) ### extent: 000000001063fa34 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.132731:0:27439:0:(osc_cache.c:582:osc_extent_merge()) ### extent: 00000000df269729 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.132738:0:27439:0:(osc_cache.c:846:osc_extent_find()) ### extent: 00000000df269729 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.132748:0:27439:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff9e35d0e392f8, count 1, dirtied 1 00000080:00200000:3.0:1787827173.132752:0:27439:0:(vvp_io.c:1237:vvp_io_write_commit()) Committed 1 pages 1024 bytes, tot: 1024 00000080:00200000:3.0:1787827173.132754:0:27439:0:(vvp_io.c:1421:vvp_io_write_start()) oosfile: write bytes 0, result: 1024 00000020:00010000:3.0:1787827173.132759:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_release():245 00000020:00010000:3.0:1787827173.132762:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1787827173.132764:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c7d81d30 (ffff9e35da704cf8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1787827173.132768:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.132777:0:27439:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 17/0,0 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1787827173.132784:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_fini():72 00000020:00010000:3.0:1787827173.132787:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c7d81d30 (ffff9e35c614ce18/1) at cl_lock_fini():72 00000080:00200000:3.0:1787827173.132791:0:27439:0:(vvp_io.c:533:vvp_io_advance()) advancing 1024 bytes 00000080:00200000:3.0:1787827173.132793:0:27439:0:(file.c:2207:ll_file_io_generic()) Range unlock [420996, 420996] 00000080:00200000:3.0:1787827173.132820:0:27439:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787827173.132824:0:27439:0:(file.c:2240:ll_file_io_generic()) oosfile: 2 io complete with rc: 0, result: 1024, restart: 0 00000080:00200000:3.0:1787827173.132830:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724400640 i_blocks 3365288 00000080:00200000:3.0:1787827173.132833:0:27439:0:(file.c:2326:ll_file_io_generic()) iot: 2, result: 1024 00000080:00200008:3.0:1787827173.132841:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724400640, count: 1024 00000080:00200000:3.0:1787827173.132846:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 420996 of 1024 to 1024 bytes 00000080:00200000:3.0:1787827173.132850:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724400640, len 1024, copied 1024 00000080:00200000:3.0:1787827173.132855:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724401664 i_blocks 3365288 00000080:00200000:3.0:1787827173.132858:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.132862:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724401664, count: 1024 00000080:00200000:3.0:1787827173.132865:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 420996 of 2048 to 1024 bytes 00000080:00200000:3.0:1787827173.132867:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724401664, len 1024, copied 1024 00000080:00200000:3.0:1787827173.132871:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724402688 i_blocks 3365288 00000080:00200000:3.0:1787827173.132873:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.132877:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724402688, count: 1024 00000080:00200000:3.0:1787827173.132880:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 420996 of 3072 to 1024 bytes 00000080:00200000:3.0:1787827173.132882:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724402688, len 1024, copied 1024 00000080:00200000:3.0:1787827173.132886:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724403712 i_blocks 3365288 00000080:00200000:3.0:1787827173.132888:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.132892:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724403712, count: 1024 00000080:00200000:3.0:1787827173.132896:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 420997 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.132898:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 0, original count 1024 00000080:00200000:3.0:1787827173.132900:0:27439:0:(file.c:2090:ll_file_io_generic()) oosfile: write ppos: 1724403712, bytes: 1024 00000080:00200000:3.0:1787827173.132902:0:27439:0:(file.c:1848:ll_io_set_mirror()) oosfile: desiginated mirror: 0 00000020:00200000:3.0:1787827173.132905:0:27439:0:(cl_io.c:218:cl_io_rw_init()) header@000000007788600a[0x110, 368550, [0x200000402:0x27:0x0] hash] 00000020:00200000:3.0:1787827173.132907:0:27439:0:(cl_io.c:218:cl_io_rw_init()) io range: 2 [1724403712, 1724404736) 0 0 00000080:00200000:3.0:1787827173.132909:0:27439:0:(vvp_io.c:1889:vvp_io_init()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1787827173.132914:0:27439:0:(file.c:2170:ll_file_io_generic()) Range lock [420997, 420997] 00020000:00200000:3.0:1787827173.132916:0:27439:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 411 chunk: [1724403712, 1724404736) 1724404736, 1024 00020000:00200000:3.0:1787827173.132932:0:27439:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1787827173.132938:0:27439:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [420997, 420997] 00000008:00010000:3.0:1787827173.132944:0:27439:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1787827173.132947:0:27439:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff9e35c7d81d30, osc lock ffff9e35c2ddcc30, flags 0x0 00010000:00010000:3.0:1787827173.132955:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.132964:0:27439:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (860377088 860381183) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.132973:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 20/0,2 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.132980:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 20/0,2 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.132987:0:27439:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 19/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.132994:0:27439:0:(vvp_io.c:1339:vvp_io_write_start()) oosfile: write [1724403712, 1724404736) append 0 append lockpos 0 00000080:00200000:3.0:1787827173.132999:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 420997 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.133019:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724403712, len 1024, copied 1024 00000080:00200000:3.0:1787827173.133024:0:27439:0:(rw26.c:933:ll_write_end()) page@000000002607a963[3 000000005ce0a915 2 1 00000000a23cbd2a 1 0 420997] 00000080:00200000:3.0:1787827173.133027:0:27439:0:(rw26.c:933:ll_write_end()) vmpage @00000000ce8e01ad fffffc0001001 3:0 ffff9e34c3d8f2b8 420997 lru 00000080:00200000:3.0:1787827173.133033:0:27439:0:(rw26.c:933:ll_write_end()) osc-page@00000000338e73d9 210053: 1< 0 - - > 2< 860377088 0 0 0x0 0x0 | 000000004db7037e 0000000052729811 > 3< 0 0 > 4< 0 0 8 484069376 - | - - + - > 5< - - + - | 0 - | 351 - -> 00000080:00200000:3.0:1787827173.133035:0:27439:0:(rw26.c:933:ll_write_end()) end page@000000002607a963 00000080:00200000:3.0:1787827173.133036:0:27439:0:(rw26.c:933:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1787827173.133038:0:27439:0:(vvp_io.c:1214:vvp_io_write_commit()) commit async pages: 1, from 0, to 1024 prio 0 00000008:00010000:3.0:1787827173.133047:0:27439:0:(osc_cache.c:565:osc_extent_merge()) ### extent: 000000001063fa34 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.133062:0:27439:0:(osc_cache.c:497:__osc_extent_remove()) ### extent: 000000001063fa34 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.133069:0:27439:0:(osc_cache.c:582:osc_extent_merge()) ### extent: 00000000df269729 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.133075:0:27439:0:(osc_cache.c:846:osc_extent_find()) ### extent: 00000000df269729 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.133086:0:27439:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff9e35d0e392f8, count 1, dirtied 1 00000080:00200000:3.0:1787827173.133090:0:27439:0:(vvp_io.c:1237:vvp_io_write_commit()) Committed 1 pages 1024 bytes, tot: 1024 00000080:00200000:3.0:1787827173.133092:0:27439:0:(vvp_io.c:1421:vvp_io_write_start()) oosfile: write bytes 0, result: 1024 00000020:00010000:3.0:1787827173.133098:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_release():245 00000020:00010000:3.0:1787827173.133100:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1787827173.133103:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c7d81d30 (ffff9e35da704cf8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1787827173.133106:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.133114:0:27439:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 17/0,0 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1787827173.133123:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_fini():72 00000020:00010000:3.0:1787827173.133126:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c7d81d30 (ffff9e35c614ce18/1) at cl_lock_fini():72 00000080:00200000:3.0:1787827173.133132:0:27439:0:(vvp_io.c:533:vvp_io_advance()) advancing 1024 bytes 00000080:00200000:3.0:1787827173.133135:0:27439:0:(file.c:2207:ll_file_io_generic()) Range unlock [420997, 420997] 00000080:00200000:3.0:1787827173.133139:0:27439:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787827173.133143:0:27439:0:(file.c:2240:ll_file_io_generic()) oosfile: 2 io complete with rc: 0, result: 1024, restart: 0 00000080:00200000:3.0:1787827173.133148:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724404736 i_blocks 3365288 00000080:00200000:3.0:1787827173.133152:0:27439:0:(file.c:2326:ll_file_io_generic()) iot: 2, result: 1024 00000080:00200008:3.0:1787827173.133163:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724404736, count: 1024 00000080:00200000:3.0:1787827173.133169:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 420997 of 1024 to 1024 bytes 00000080:00200000:3.0:1787827173.133173:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724404736, len 1024, copied 1024 00000080:00200000:3.0:1787827173.133181:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724405760 i_blocks 3365288 00000080:00200000:3.0:1787827173.133187:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.133195:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724405760, count: 1024 00000080:00200000:3.0:1787827173.133203:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 420997 of 2048 to 1024 bytes 00000080:00200000:3.0:1787827173.133210:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724405760, len 1024, copied 1024 00000080:00200000:3.0:1787827173.133219:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724406784 i_blocks 3365288 00000080:00200000:3.0:1787827173.133223:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.133228:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724406784, count: 1024 00000080:00200000:3.0:1787827173.133232:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 420997 of 3072 to 1024 bytes 00000080:00200000:3.0:1787827173.133237:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724406784, len 1024, copied 1024 00000080:00200000:3.0:1787827173.133246:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724407808 i_blocks 3365288 00000080:00200000:3.0:1787827173.133250:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.133256:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724407808, count: 1024 00000080:00200000:3.0:1787827173.133261:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 420998 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.133264:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 0, original count 1024 00000080:00200000:3.0:1787827173.133268:0:27439:0:(file.c:2090:ll_file_io_generic()) oosfile: write ppos: 1724407808, bytes: 1024 00000080:00200000:3.0:1787827173.133274:0:27439:0:(file.c:1848:ll_io_set_mirror()) oosfile: desiginated mirror: 0 00000020:00200000:3.0:1787827173.133278:0:27439:0:(cl_io.c:218:cl_io_rw_init()) header@000000007788600a[0x110, 368551, [0x200000402:0x27:0x0] hash] 00000020:00200000:3.0:1787827173.133280:0:27439:0:(cl_io.c:218:cl_io_rw_init()) io range: 2 [1724407808, 1724408832) 0 0 00000080:00200000:3.0:1787827173.133287:0:27439:0:(vvp_io.c:1889:vvp_io_init()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1787827173.133299:0:27439:0:(file.c:2170:ll_file_io_generic()) Range lock [420998, 420998] 00020000:00200000:3.0:1787827173.133306:0:27439:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 411 chunk: [1724407808, 1724408832) 1724408832, 1024 00020000:00200000:3.0:1787827173.133313:0:27439:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1787827173.133321:0:27439:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [420998, 420998] 00000008:00010000:3.0:1787827173.133328:0:27439:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1787827173.133332:0:27439:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff9e35c7d81d30, osc lock ffff9e35c2ddcc30, flags 0x0 00010000:00010000:3.0:1787827173.133346:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.133362:0:27439:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (860381184 860385279) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.133373:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 20/0,2 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.133381:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 20/0,2 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.133389:0:27439:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 19/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.133398:0:27439:0:(vvp_io.c:1339:vvp_io_write_start()) oosfile: write [1724407808, 1724408832) append 0 append lockpos 0 00000080:00200000:3.0:1787827173.133403:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 420998 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.133422:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724407808, len 1024, copied 1024 00000080:00200000:3.0:1787827173.133428:0:27439:0:(rw26.c:933:ll_write_end()) page@00000000ad6f2d86[3 000000005ce0a915 2 1 00000000a23cbd2a 1 0 420998] 00000080:00200000:3.0:1787827173.133431:0:27439:0:(rw26.c:933:ll_write_end()) vmpage @000000004030fe84 fffffc0001001 3:0 ffff9e34c3d8f488 420998 lru 00000080:00200000:3.0:1787827173.133436:0:27439:0:(rw26.c:933:ll_write_end()) osc-page@000000000b83f4f2 210054: 1< 0 - - > 2< 860381184 0 0 0x0 0x0 | 000000004db7037e 0000000052729811 > 3< 0 0 > 4< 0 0 8 484065280 - | - - + - > 5< - - + - | 0 - | 352 - -> 00000080:00200000:3.0:1787827173.133439:0:27439:0:(rw26.c:933:ll_write_end()) end page@00000000ad6f2d86 00000080:00200000:3.0:1787827173.133440:0:27439:0:(rw26.c:933:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1787827173.133443:0:27439:0:(vvp_io.c:1214:vvp_io_write_commit()) commit async pages: 1, from 0, to 1024 prio 0 00000008:00010000:3.0:1787827173.133452:0:27439:0:(osc_cache.c:565:osc_extent_merge()) ### extent: 000000001063fa34 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.133466:0:27439:0:(osc_cache.c:497:__osc_extent_remove()) ### extent: 000000001063fa34 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.133474:0:27439:0:(osc_cache.c:582:osc_extent_merge()) ### extent: 00000000df269729 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.133481:0:27439:0:(osc_cache.c:846:osc_extent_find()) ### extent: 00000000df269729 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.133493:0:27439:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff9e35d0e392f8, count 1, dirtied 1 00000080:00200000:3.0:1787827173.133496:0:27439:0:(vvp_io.c:1237:vvp_io_write_commit()) Committed 1 pages 1024 bytes, tot: 1024 00000080:00200000:3.0:1787827173.133498:0:27439:0:(vvp_io.c:1421:vvp_io_write_start()) oosfile: write bytes 0, result: 1024 00000020:00010000:3.0:1787827173.133504:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_release():245 00000020:00010000:3.0:1787827173.133507:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1787827173.133510:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c7d81d30 (ffff9e35da704cf8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1787827173.133514:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.133523:0:27439:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 17/0,0 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1787827173.133533:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_fini():72 00000020:00010000:3.0:1787827173.133535:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c7d81d30 (ffff9e35c614ce18/1) at cl_lock_fini():72 00000080:00200000:3.0:1787827173.133540:0:27439:0:(vvp_io.c:533:vvp_io_advance()) advancing 1024 bytes 00000080:00200000:3.0:1787827173.133543:0:27439:0:(file.c:2207:ll_file_io_generic()) Range unlock [420998, 420998] 00000080:00200000:3.0:1787827173.133547:0:27439:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787827173.133550:0:27439:0:(file.c:2240:ll_file_io_generic()) oosfile: 2 io complete with rc: 0, result: 1024, restart: 0 00000080:00200000:3.0:1787827173.133555:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724408832 i_blocks 3365288 00000080:00200000:3.0:1787827173.133557:0:27439:0:(file.c:2326:ll_file_io_generic()) iot: 2, result: 1024 00000080:00200008:3.0:1787827173.133565:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724408832, count: 1024 00000080:00200000:3.0:1787827173.133570:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 420998 of 1024 to 1024 bytes 00000080:00200000:3.0:1787827173.133573:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724408832, len 1024, copied 1024 00000080:00200000:3.0:1787827173.133578:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724409856 i_blocks 3365288 00000080:00200000:3.0:1787827173.133580:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.133585:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724409856, count: 1024 00000080:00200000:3.0:1787827173.133589:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 420998 of 2048 to 1024 bytes 00000080:00200000:3.0:1787827173.133591:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724409856, len 1024, copied 1024 00000080:00200000:3.0:1787827173.133595:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724410880 i_blocks 3365288 00000080:00200000:3.0:1787827173.133597:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.133601:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724410880, count: 1024 00000080:00200000:3.0:1787827173.133605:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 420998 of 3072 to 1024 bytes 00000080:00200000:3.0:1787827173.133607:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724410880, len 1024, copied 1024 00000080:00200000:3.0:1787827173.133612:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724411904 i_blocks 3365288 00000080:00200000:3.0:1787827173.133614:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.133619:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724411904, count: 1024 00000080:00200000:3.0:1787827173.133623:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 420999 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.133625:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 0, original count 1024 00000080:00200000:3.0:1787827173.133627:0:27439:0:(file.c:2090:ll_file_io_generic()) oosfile: write ppos: 1724411904, bytes: 1024 00000080:00200000:3.0:1787827173.133629:0:27439:0:(file.c:1848:ll_io_set_mirror()) oosfile: desiginated mirror: 0 00000020:00200000:3.0:1787827173.133633:0:27439:0:(cl_io.c:218:cl_io_rw_init()) header@000000007788600a[0x110, 368552, [0x200000402:0x27:0x0] hash] 00000020:00200000:3.0:1787827173.133635:0:27439:0:(cl_io.c:218:cl_io_rw_init()) io range: 2 [1724411904, 1724412928) 0 0 00000080:00200000:3.0:1787827173.133638:0:27439:0:(vvp_io.c:1889:vvp_io_init()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1787827173.133643:0:27439:0:(file.c:2170:ll_file_io_generic()) Range lock [420999, 420999] 00020000:00200000:3.0:1787827173.133646:0:27439:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 411 chunk: [1724411904, 1724412928) 1724412928, 1024 00020000:00200000:3.0:1787827173.133649:0:27439:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1787827173.133655:0:27439:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [420999, 420999] 00000008:00010000:3.0:1787827173.133661:0:27439:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1787827173.133663:0:27439:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff9e35c7d81d30, osc lock ffff9e35c2ddcc30, flags 0x0 00010000:00010000:3.0:1787827173.133671:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.133680:0:27439:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (860385280 860389375) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.133688:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 20/0,2 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.133696:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 20/0,2 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.133704:0:27439:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 19/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.133712:0:27439:0:(vvp_io.c:1339:vvp_io_write_start()) oosfile: write [1724411904, 1724412928) append 0 append lockpos 0 00000080:00200000:3.0:1787827173.133717:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 420999 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.133748:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724411904, len 1024, copied 1024 00000080:00200000:3.0:1787827173.133753:0:27439:0:(rw26.c:933:ll_write_end()) page@00000000a78ff306[3 000000005ce0a915 2 1 00000000a23cbd2a 1 0 420999] 00000080:00200000:3.0:1787827173.133756:0:27439:0:(rw26.c:933:ll_write_end()) vmpage @000000000e6969a2 fffffc0001001 3:0 ffff9e34c3cb0d98 420999 lru 00000080:00200000:3.0:1787827173.133762:0:27439:0:(rw26.c:933:ll_write_end()) osc-page@000000007924896a 210055: 1< 0 - - > 2< 860385280 0 0 0x0 0x0 | 000000004db7037e 0000000052729811 > 3< 0 0 > 4< 0 0 8 484061184 - | - - + - > 5< - - + - | 0 - | 353 - -> 00000080:00200000:3.0:1787827173.133764:0:27439:0:(rw26.c:933:ll_write_end()) end page@00000000a78ff306 00000080:00200000:3.0:1787827173.133765:0:27439:0:(rw26.c:933:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1787827173.133767:0:27439:0:(vvp_io.c:1214:vvp_io_write_commit()) commit async pages: 1, from 0, to 1024 prio 0 00000008:00010000:3.0:1787827173.133776:0:27439:0:(osc_cache.c:565:osc_extent_merge()) ### extent: 000000001063fa34 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.133790:0:27439:0:(osc_cache.c:497:__osc_extent_remove()) ### extent: 000000001063fa34 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.133820:0:27439:0:(osc_cache.c:582:osc_extent_merge()) ### extent: 00000000df269729 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.133830:0:27439:0:(osc_cache.c:846:osc_extent_find()) ### extent: 00000000df269729 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.133843:0:27439:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff9e35d0e392f8, count 1, dirtied 1 00000080:00200000:3.0:1787827173.133847:0:27439:0:(vvp_io.c:1237:vvp_io_write_commit()) Committed 1 pages 1024 bytes, tot: 1024 00000080:00200000:3.0:1787827173.133850:0:27439:0:(vvp_io.c:1421:vvp_io_write_start()) oosfile: write bytes 0, result: 1024 00000020:00010000:3.0:1787827173.133857:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_release():245 00000020:00010000:3.0:1787827173.133860:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1787827173.133862:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c7d81d30 (ffff9e35da704cf8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1787827173.133866:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.133875:0:27439:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 17/0,0 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1787827173.133890:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_fini():72 00000020:00010000:3.0:1787827173.133894:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c7d81d30 (ffff9e35c614ce18/1) at cl_lock_fini():72 00000080:00200000:3.0:1787827173.133899:0:27439:0:(vvp_io.c:533:vvp_io_advance()) advancing 1024 bytes 00000080:00200000:3.0:1787827173.133903:0:27439:0:(file.c:2207:ll_file_io_generic()) Range unlock [420999, 420999] 00000080:00200000:3.0:1787827173.133908:0:27439:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787827173.133911:0:27439:0:(file.c:2240:ll_file_io_generic()) oosfile: 2 io complete with rc: 0, result: 1024, restart: 0 00000080:00200000:3.0:1787827173.133916:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724412928 i_blocks 3365288 00000080:00200000:3.0:1787827173.133934:0:27439:0:(file.c:2326:ll_file_io_generic()) iot: 2, result: 1024 00000080:00200008:3.0:1787827173.133944:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724412928, count: 1024 00000080:00200000:3.0:1787827173.133951:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 420999 of 1024 to 1024 bytes 00000080:00200000:3.0:1787827173.133958:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724412928, len 1024, copied 1024 00000080:00200000:3.0:1787827173.133966:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724413952 i_blocks 3365288 00000080:00200000:3.0:1787827173.133969:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.133975:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724413952, count: 1024 00000080:00200000:3.0:1787827173.133979:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 420999 of 2048 to 1024 bytes 00000080:00200000:3.0:1787827173.133982:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724413952, len 1024, copied 1024 00000080:00200000:3.0:1787827173.133986:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724414976 i_blocks 3365288 00000080:00200000:3.0:1787827173.133989:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.133993:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724414976, count: 1024 00000080:00200000:3.0:1787827173.133997:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 420999 of 3072 to 1024 bytes 00000080:00200000:3.0:1787827173.134000:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724414976, len 1024, copied 1024 00000080:00200000:3.0:1787827173.134004:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724416000 i_blocks 3365288 00000080:00200000:3.0:1787827173.134007:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.134012:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724416000, count: 1024 00000080:00200000:3.0:1787827173.134016:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421000 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.134019:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 0, original count 1024 00000080:00200000:3.0:1787827173.134021:0:27439:0:(file.c:2090:ll_file_io_generic()) oosfile: write ppos: 1724416000, bytes: 1024 00000080:00200000:3.0:1787827173.134024:0:27439:0:(file.c:1848:ll_io_set_mirror()) oosfile: desiginated mirror: 0 00000020:00200000:3.0:1787827173.134028:0:27439:0:(cl_io.c:218:cl_io_rw_init()) header@000000007788600a[0x110, 368553, [0x200000402:0x27:0x0] hash] 00000020:00200000:3.0:1787827173.134030:0:27439:0:(cl_io.c:218:cl_io_rw_init()) io range: 2 [1724416000, 1724417024) 0 0 00000080:00200000:3.0:1787827173.134033:0:27439:0:(vvp_io.c:1889:vvp_io_init()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1787827173.134039:0:27439:0:(file.c:2170:ll_file_io_generic()) Range lock [421000, 421000] 00020000:00200000:3.0:1787827173.134042:0:27439:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 411 chunk: [1724416000, 1724417024) 1724417024, 1024 00020000:00200000:3.0:1787827173.134045:0:27439:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1787827173.134052:0:27439:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [421000, 421000] 00000008:00010000:3.0:1787827173.134059:0:27439:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1787827173.134062:0:27439:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff9e35c7d81d30, osc lock ffff9e35c2ddcc30, flags 0x0 00010000:00010000:3.0:1787827173.134071:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.134081:0:27439:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (860389376 860393471) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.134090:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 20/0,2 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.134099:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 20/0,2 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.134106:0:27439:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 19/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.134113:0:27439:0:(vvp_io.c:1339:vvp_io_write_start()) oosfile: write [1724416000, 1724417024) append 0 append lockpos 0 00000080:00200000:3.0:1787827173.134119:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421000 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.134138:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724416000, len 1024, copied 1024 00000080:00200000:3.0:1787827173.134143:0:27439:0:(rw26.c:933:ll_write_end()) page@0000000066080658[3 000000005ce0a915 2 1 00000000a23cbd2a 1 0 421000] 00000080:00200000:3.0:1787827173.134146:0:27439:0:(rw26.c:933:ll_write_end()) vmpage @000000005c369f9a fffffc0001001 3:0 ffff9e34c3cb0000 421000 lru 00000080:00200000:3.0:1787827173.134151:0:27439:0:(rw26.c:933:ll_write_end()) osc-page@000000003f3a6582 210056: 1< 0 - - > 2< 860389376 0 0 0x0 0x0 | 000000004db7037e 0000000052729811 > 3< 0 0 > 4< 0 0 8 484057088 - | - - + - > 5< - - + - | 0 - | 354 - -> 00000080:00200000:3.0:1787827173.134153:0:27439:0:(rw26.c:933:ll_write_end()) end page@0000000066080658 00000080:00200000:3.0:1787827173.134154:0:27439:0:(rw26.c:933:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1787827173.134157:0:27439:0:(vvp_io.c:1214:vvp_io_write_commit()) commit async pages: 1, from 0, to 1024 prio 0 00000008:00010000:3.0:1787827173.134166:0:27439:0:(osc_cache.c:565:osc_extent_merge()) ### extent: 000000001063fa34 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.134181:0:27439:0:(osc_cache.c:497:__osc_extent_remove()) ### extent: 000000001063fa34 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.134189:0:27439:0:(osc_cache.c:582:osc_extent_merge()) ### extent: 00000000df269729 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.134197:0:27439:0:(osc_cache.c:846:osc_extent_find()) ### extent: 00000000df269729 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.134210:0:27439:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff9e35d0e392f8, count 1, dirtied 1 00000080:00200000:3.0:1787827173.134214:0:27439:0:(vvp_io.c:1237:vvp_io_write_commit()) Committed 1 pages 1024 bytes, tot: 1024 00000080:00200000:3.0:1787827173.134216:0:27439:0:(vvp_io.c:1421:vvp_io_write_start()) oosfile: write bytes 0, result: 1024 00000020:00010000:3.0:1787827173.134222:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_release():245 00000020:00010000:3.0:1787827173.134224:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1787827173.134227:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c7d81d30 (ffff9e35da704cf8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1787827173.134232:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.134241:0:27439:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 17/0,0 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1787827173.134309:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_fini():72 00000020:00010000:3.0:1787827173.134312:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c7d81d30 (ffff9e35c614ce18/1) at cl_lock_fini():72 00000080:00200000:3.0:1787827173.134316:0:27439:0:(vvp_io.c:533:vvp_io_advance()) advancing 1024 bytes 00000080:00200000:3.0:1787827173.134319:0:27439:0:(file.c:2207:ll_file_io_generic()) Range unlock [421000, 421000] 00000080:00200000:3.0:1787827173.134322:0:27439:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787827173.134325:0:27439:0:(file.c:2240:ll_file_io_generic()) oosfile: 2 io complete with rc: 0, result: 1024, restart: 0 00000080:00200000:3.0:1787827173.134330:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724417024 i_blocks 3365288 00000080:00200000:3.0:1787827173.134332:0:27439:0:(file.c:2326:ll_file_io_generic()) iot: 2, result: 1024 00000080:00200008:3.0:1787827173.134340:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724417024, count: 1024 00000080:00200000:3.0:1787827173.134345:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421000 of 1024 to 1024 bytes 00000080:00200000:3.0:1787827173.134348:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724417024, len 1024, copied 1024 00000080:00200000:3.0:1787827173.134353:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724418048 i_blocks 3365288 00000080:00200000:3.0:1787827173.134355:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.134359:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724418048, count: 1024 00000080:00200000:3.0:1787827173.134363:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421000 of 2048 to 1024 bytes 00000080:00200000:3.0:1787827173.134365:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724418048, len 1024, copied 1024 00000080:00200000:3.0:1787827173.134369:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724419072 i_blocks 3365288 00000080:00200000:3.0:1787827173.134371:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.134374:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724419072, count: 1024 00000080:00200000:3.0:1787827173.134378:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421000 of 3072 to 1024 bytes 00000080:00200000:3.0:1787827173.134380:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724419072, len 1024, copied 1024 00000080:00200000:3.0:1787827173.134384:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724420096 i_blocks 3365288 00000080:00200000:3.0:1787827173.134386:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.134390:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724420096, count: 1024 00000080:00200000:3.0:1787827173.134393:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421001 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.134395:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 0, original count 1024 00000080:00200000:3.0:1787827173.134397:0:27439:0:(file.c:2090:ll_file_io_generic()) oosfile: write ppos: 1724420096, bytes: 1024 00000080:00200000:3.0:1787827173.134400:0:27439:0:(file.c:1848:ll_io_set_mirror()) oosfile: desiginated mirror: 0 00000020:00200000:3.0:1787827173.134403:0:27439:0:(cl_io.c:218:cl_io_rw_init()) header@000000007788600a[0x110, 368554, [0x200000402:0x27:0x0] hash] 00000020:00200000:3.0:1787827173.134405:0:27439:0:(cl_io.c:218:cl_io_rw_init()) io range: 2 [1724420096, 1724421120) 0 0 00000080:00200000:3.0:1787827173.134408:0:27439:0:(vvp_io.c:1889:vvp_io_init()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1787827173.134413:0:27439:0:(file.c:2170:ll_file_io_generic()) Range lock [421001, 421001] 00020000:00200000:3.0:1787827173.134416:0:27439:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 411 chunk: [1724420096, 1724421120) 1724421120, 1024 00020000:00200000:3.0:1787827173.134418:0:27439:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1787827173.134423:0:27439:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [421001, 421001] 00000008:00010000:3.0:1787827173.134428:0:27439:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1787827173.134431:0:27439:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff9e35c7d81d30, osc lock ffff9e35c2ddcc30, flags 0x0 00010000:00010000:3.0:1787827173.134438:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.134447:0:27439:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (860393472 860397567) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.134456:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 20/0,2 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.134464:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 20/0,2 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.134473:0:27439:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 19/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.134480:0:27439:0:(vvp_io.c:1339:vvp_io_write_start()) oosfile: write [1724420096, 1724421120) append 0 append lockpos 0 00000080:00200000:3.0:1787827173.134485:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421001 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.134500:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724420096, len 1024, copied 1024 00000080:00200000:3.0:1787827173.134505:0:27439:0:(rw26.c:933:ll_write_end()) page@0000000082d67431[3 000000005ce0a915 2 1 00000000a23cbd2a 1 0 421001] 00000080:00200000:3.0:1787827173.134509:0:27439:0:(rw26.c:933:ll_write_end()) vmpage @00000000e6abb9c9 fffffc0001001 3:0 ffff9e34c3cb02b8 421001 lru 00000080:00200000:3.0:1787827173.134514:0:27439:0:(rw26.c:933:ll_write_end()) osc-page@0000000054c53481 210057: 1< 0 - - > 2< 860393472 0 0 0x0 0x0 | 000000004db7037e 0000000052729811 > 3< 0 0 > 4< 0 0 8 484052992 - | - - + - > 5< - - + - | 0 - | 355 - -> 00000080:00200000:3.0:1787827173.134516:0:27439:0:(rw26.c:933:ll_write_end()) end page@0000000082d67431 00000080:00200000:3.0:1787827173.134517:0:27439:0:(rw26.c:933:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1787827173.134519:0:27439:0:(vvp_io.c:1214:vvp_io_write_commit()) commit async pages: 1, from 0, to 1024 prio 0 00000008:00010000:3.0:1787827173.134528:0:27439:0:(osc_cache.c:565:osc_extent_merge()) ### extent: 000000001063fa34 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.134541:0:27439:0:(osc_cache.c:497:__osc_extent_remove()) ### extent: 000000001063fa34 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.134548:0:27439:0:(osc_cache.c:582:osc_extent_merge()) ### extent: 00000000df269729 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.134556:0:27439:0:(osc_cache.c:846:osc_extent_find()) ### extent: 00000000df269729 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.134567:0:27439:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff9e35d0e392f8, count 1, dirtied 1 00000080:00200000:3.0:1787827173.134570:0:27439:0:(vvp_io.c:1237:vvp_io_write_commit()) Committed 1 pages 1024 bytes, tot: 1024 00000080:00200000:3.0:1787827173.134572:0:27439:0:(vvp_io.c:1421:vvp_io_write_start()) oosfile: write bytes 0, result: 1024 00000020:00010000:3.0:1787827173.134579:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_release():245 00000020:00010000:3.0:1787827173.134581:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1787827173.134584:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c7d81d30 (ffff9e35da704cf8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1787827173.134588:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.134597:0:27439:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 17/0,0 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1787827173.134606:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_fini():72 00000020:00010000:3.0:1787827173.134609:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c7d81d30 (ffff9e35c614ce18/1) at cl_lock_fini():72 00000080:00200000:3.0:1787827173.134613:0:27439:0:(vvp_io.c:533:vvp_io_advance()) advancing 1024 bytes 00000080:00200000:3.0:1787827173.134616:0:27439:0:(file.c:2207:ll_file_io_generic()) Range unlock [421001, 421001] 00000080:00200000:3.0:1787827173.134620:0:27439:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787827173.134623:0:27439:0:(file.c:2240:ll_file_io_generic()) oosfile: 2 io complete with rc: 0, result: 1024, restart: 0 00000080:00200000:3.0:1787827173.134628:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724421120 i_blocks 3365288 00000080:00200000:3.0:1787827173.134630:0:27439:0:(file.c:2326:ll_file_io_generic()) iot: 2, result: 1024 00000080:00200008:3.0:1787827173.134639:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724421120, count: 1024 00000080:00200000:3.0:1787827173.134644:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421001 of 1024 to 1024 bytes 00000080:00200000:3.0:1787827173.134647:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724421120, len 1024, copied 1024 00000080:00200000:3.0:1787827173.134652:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724422144 i_blocks 3365288 00000080:00200000:3.0:1787827173.134655:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.134659:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724422144, count: 1024 00000080:00200000:3.0:1787827173.134664:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421001 of 2048 to 1024 bytes 00000080:00200000:3.0:1787827173.134666:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724422144, len 1024, copied 1024 00000080:00200000:3.0:1787827173.134670:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724423168 i_blocks 3365288 00000080:00200000:3.0:1787827173.134673:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.134677:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724423168, count: 1024 00000080:00200000:3.0:1787827173.134680:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421001 of 3072 to 1024 bytes 00000080:00200000:3.0:1787827173.134683:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724423168, len 1024, copied 1024 00000080:00200000:3.0:1787827173.134687:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724424192 i_blocks 3365288 00000080:00200000:3.0:1787827173.134689:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.134694:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724424192, count: 1024 00000080:00200000:3.0:1787827173.134697:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421002 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.134699:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 0, original count 1024 00000080:00200000:3.0:1787827173.134702:0:27439:0:(file.c:2090:ll_file_io_generic()) oosfile: write ppos: 1724424192, bytes: 1024 00000080:00200000:3.0:1787827173.134704:0:27439:0:(file.c:1848:ll_io_set_mirror()) oosfile: desiginated mirror: 0 00000020:00200000:3.0:1787827173.134708:0:27439:0:(cl_io.c:218:cl_io_rw_init()) header@000000007788600a[0x110, 368555, [0x200000402:0x27:0x0] hash] 00000020:00200000:3.0:1787827173.134710:0:27439:0:(cl_io.c:218:cl_io_rw_init()) io range: 2 [1724424192, 1724425216) 0 0 00000080:00200000:3.0:1787827173.134712:0:27439:0:(vvp_io.c:1889:vvp_io_init()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1787827173.134718:0:27439:0:(file.c:2170:ll_file_io_generic()) Range lock [421002, 421002] 00020000:00200000:3.0:1787827173.134721:0:27439:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 411 chunk: [1724424192, 1724425216) 1724425216, 1024 00020000:00200000:3.0:1787827173.134723:0:27439:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1787827173.134729:0:27439:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [421002, 421002] 00000008:00010000:3.0:1787827173.134735:0:27439:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1787827173.134737:0:27439:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff9e35c7d81d30, osc lock ffff9e35c2ddcc30, flags 0x0 00010000:00010000:3.0:1787827173.134745:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.134754:0:27439:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (860397568 860401663) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.134763:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 20/0,2 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.134771:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 20/0,2 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.134779:0:27439:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 19/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.134787:0:27439:0:(vvp_io.c:1339:vvp_io_write_start()) oosfile: write [1724424192, 1724425216) append 0 append lockpos 0 00000080:00200000:3.0:1787827173.134792:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421002 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.134831:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724424192, len 1024, copied 1024 00000080:00200000:3.0:1787827173.134836:0:27439:0:(rw26.c:933:ll_write_end()) page@00000000c0eecc04[3 000000005ce0a915 2 1 00000000a23cbd2a 1 0 421002] 00000080:00200000:3.0:1787827173.134840:0:27439:0:(rw26.c:933:ll_write_end()) vmpage @00000000af7899ad fffffc0001001 3:0 ffff9e34c3cb0488 421002 lru 00000080:00200000:3.0:1787827173.134846:0:27439:0:(rw26.c:933:ll_write_end()) osc-page@0000000081fdb49f 210058: 1< 0 - - > 2< 860397568 0 0 0x0 0x0 | 000000004db7037e 0000000052729811 > 3< 0 0 > 4< 0 0 8 484048896 - | - - + - > 5< - - + - | 0 - | 356 - -> 00000080:00200000:3.0:1787827173.134848:0:27439:0:(rw26.c:933:ll_write_end()) end page@00000000c0eecc04 00000080:00200000:3.0:1787827173.134849:0:27439:0:(rw26.c:933:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1787827173.134852:0:27439:0:(vvp_io.c:1214:vvp_io_write_commit()) commit async pages: 1, from 0, to 1024 prio 0 00000008:00010000:3.0:1787827173.134861:0:27439:0:(osc_cache.c:565:osc_extent_merge()) ### extent: 000000001063fa34 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.134875:0:27439:0:(osc_cache.c:497:__osc_extent_remove()) ### extent: 000000001063fa34 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.134882:0:27439:0:(osc_cache.c:582:osc_extent_merge()) ### extent: 00000000df269729 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.134890:0:27439:0:(osc_cache.c:846:osc_extent_find()) ### extent: 00000000df269729 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.134901:0:27439:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff9e35d0e392f8, count 1, dirtied 1 00000080:00200000:3.0:1787827173.134905:0:27439:0:(vvp_io.c:1237:vvp_io_write_commit()) Committed 1 pages 1024 bytes, tot: 1024 00000080:00200000:3.0:1787827173.134907:0:27439:0:(vvp_io.c:1421:vvp_io_write_start()) oosfile: write bytes 0, result: 1024 00000020:00010000:3.0:1787827173.134914:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_release():245 00000020:00010000:3.0:1787827173.134916:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1787827173.134934:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c7d81d30 (ffff9e35da704cf8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1787827173.134939:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.134947:0:27439:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 17/0,0 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1787827173.134956:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_fini():72 00000020:00010000:3.0:1787827173.134959:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c7d81d30 (ffff9e35c614ce18/1) at cl_lock_fini():72 00000080:00200000:3.0:1787827173.134963:0:27439:0:(vvp_io.c:533:vvp_io_advance()) advancing 1024 bytes 00000080:00200000:3.0:1787827173.134966:0:27439:0:(file.c:2207:ll_file_io_generic()) Range unlock [421002, 421002] 00000080:00200000:3.0:1787827173.134971:0:27439:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787827173.134974:0:27439:0:(file.c:2240:ll_file_io_generic()) oosfile: 2 io complete with rc: 0, result: 1024, restart: 0 00000080:00200000:3.0:1787827173.134979:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724425216 i_blocks 3365288 00000080:00200000:3.0:1787827173.134982:0:27439:0:(file.c:2326:ll_file_io_generic()) iot: 2, result: 1024 00000080:00200008:3.0:1787827173.134989:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724425216, count: 1024 00000080:00200000:3.0:1787827173.134994:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421002 of 1024 to 1024 bytes 00000080:00200000:3.0:1787827173.134997:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724425216, len 1024, copied 1024 00000080:00200000:3.0:1787827173.135003:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724426240 i_blocks 3365288 00000080:00200000:3.0:1787827173.135005:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.135009:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724426240, count: 1024 00000080:00200000:3.0:1787827173.135013:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421002 of 2048 to 1024 bytes 00000080:00200000:3.0:1787827173.135016:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724426240, len 1024, copied 1024 00000080:00200000:3.0:1787827173.135020:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724427264 i_blocks 3365288 00000080:00200000:3.0:1787827173.135023:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.135027:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724427264, count: 1024 00000080:00200000:3.0:1787827173.135030:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421002 of 3072 to 1024 bytes 00000080:00200000:3.0:1787827173.135033:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724427264, len 1024, copied 1024 00000080:00200000:3.0:1787827173.135037:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724428288 i_blocks 3365288 00000080:00200000:3.0:1787827173.135039:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.135043:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724428288, count: 1024 00000080:00200000:3.0:1787827173.135047:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421003 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.135049:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 0, original count 1024 00000080:00200000:3.0:1787827173.135051:0:27439:0:(file.c:2090:ll_file_io_generic()) oosfile: write ppos: 1724428288, bytes: 1024 00000080:00200000:3.0:1787827173.135053:0:27439:0:(file.c:1848:ll_io_set_mirror()) oosfile: desiginated mirror: 0 00000020:00200000:3.0:1787827173.135057:0:27439:0:(cl_io.c:218:cl_io_rw_init()) header@000000007788600a[0x110, 368556, [0x200000402:0x27:0x0] hash] 00000020:00200000:3.0:1787827173.135058:0:27439:0:(cl_io.c:218:cl_io_rw_init()) io range: 2 [1724428288, 1724429312) 0 0 00000080:00200000:3.0:1787827173.135060:0:27439:0:(vvp_io.c:1889:vvp_io_init()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1787827173.135066:0:27439:0:(file.c:2170:ll_file_io_generic()) Range lock [421003, 421003] 00020000:00200000:3.0:1787827173.135068:0:27439:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 411 chunk: [1724428288, 1724429312) 1724429312, 1024 00020000:00200000:3.0:1787827173.135071:0:27439:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1787827173.135076:0:27439:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [421003, 421003] 00000008:00010000:3.0:1787827173.135081:0:27439:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1787827173.135084:0:27439:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff9e35c7d81d30, osc lock ffff9e35c2ddcc30, flags 0x0 00010000:00010000:3.0:1787827173.135092:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.135100:0:27439:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (860401664 860405759) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.135109:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 20/0,2 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.135116:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 20/0,2 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.135124:0:27439:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 19/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.135131:0:27439:0:(vvp_io.c:1339:vvp_io_write_start()) oosfile: write [1724428288, 1724429312) append 0 append lockpos 0 00000080:00200000:3.0:1787827173.135137:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421003 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.135154:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724428288, len 1024, copied 1024 00000080:00200000:3.0:1787827173.135159:0:27439:0:(rw26.c:933:ll_write_end()) page@00000000ba786c69[3 000000005ce0a915 2 1 00000000a23cbd2a 1 0 421003] 00000080:00200000:3.0:1787827173.135161:0:27439:0:(rw26.c:933:ll_write_end()) vmpage @0000000046f63aa2 fffffc0001001 3:0 ffff9e34c3cb0bc8 421003 lru 00000080:00200000:3.0:1787827173.135167:0:27439:0:(rw26.c:933:ll_write_end()) osc-page@00000000212c9a6a 210059: 1< 0 - - > 2< 860401664 0 0 0x0 0x0 | 000000004db7037e 0000000052729811 > 3< 0 0 > 4< 0 0 8 484044800 - | - - + - > 5< - - + - | 0 - | 357 - -> 00000080:00200000:3.0:1787827173.135169:0:27439:0:(rw26.c:933:ll_write_end()) end page@00000000ba786c69 00000080:00200000:3.0:1787827173.135170:0:27439:0:(rw26.c:933:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1787827173.135172:0:27439:0:(vvp_io.c:1214:vvp_io_write_commit()) commit async pages: 1, from 0, to 1024 prio 0 00000008:00010000:3.0:1787827173.135180:0:27439:0:(osc_cache.c:565:osc_extent_merge()) ### extent: 000000001063fa34 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.135193:0:27439:0:(osc_cache.c:497:__osc_extent_remove()) ### extent: 000000001063fa34 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.135200:0:27439:0:(osc_cache.c:582:osc_extent_merge()) ### extent: 00000000df269729 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.135208:0:27439:0:(osc_cache.c:846:osc_extent_find()) ### extent: 00000000df269729 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.135219:0:27439:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff9e35d0e392f8, count 1, dirtied 1 00000080:00200000:3.0:1787827173.135222:0:27439:0:(vvp_io.c:1237:vvp_io_write_commit()) Committed 1 pages 1024 bytes, tot: 1024 00000080:00200000:3.0:1787827173.135225:0:27439:0:(vvp_io.c:1421:vvp_io_write_start()) oosfile: write bytes 0, result: 1024 00000020:00010000:3.0:1787827173.135230:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_release():245 00000020:00010000:3.0:1787827173.135233:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1787827173.135236:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c7d81d30 (ffff9e35da704cf8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1787827173.135240:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.135248:0:27439:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 17/0,0 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1787827173.135257:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_fini():72 00000020:00010000:3.0:1787827173.135260:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c7d81d30 (ffff9e35c614ce18/1) at cl_lock_fini():72 00000080:00200000:3.0:1787827173.135264:0:27439:0:(vvp_io.c:533:vvp_io_advance()) advancing 1024 bytes 00000080:00200000:3.0:1787827173.135267:0:27439:0:(file.c:2207:ll_file_io_generic()) Range unlock [421003, 421003] 00000080:00200000:3.0:1787827173.135271:0:27439:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787827173.135274:0:27439:0:(file.c:2240:ll_file_io_generic()) oosfile: 2 io complete with rc: 0, result: 1024, restart: 0 00000080:00200000:3.0:1787827173.135279:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724429312 i_blocks 3365288 00000080:00200000:3.0:1787827173.135281:0:27439:0:(file.c:2326:ll_file_io_generic()) iot: 2, result: 1024 00000080:00200008:3.0:1787827173.135288:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724429312, count: 1024 00000080:00200000:3.0:1787827173.135292:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421003 of 1024 to 1024 bytes 00000080:00200000:3.0:1787827173.135295:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724429312, len 1024, copied 1024 00000080:00200000:3.0:1787827173.135300:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724430336 i_blocks 3365288 00000080:00200000:3.0:1787827173.135302:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.135305:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724430336, count: 1024 00000080:00200000:3.0:1787827173.135308:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421003 of 2048 to 1024 bytes 00000080:00200000:3.0:1787827173.135310:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724430336, len 1024, copied 1024 00000080:00200000:3.0:1787827173.135314:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724431360 i_blocks 3365288 00000080:00200000:3.0:1787827173.135316:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.135319:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724431360, count: 1024 00000080:00200000:3.0:1787827173.135322:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421003 of 3072 to 1024 bytes 00000080:00200000:3.0:1787827173.135324:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724431360, len 1024, copied 1024 00000080:00200000:3.0:1787827173.135327:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724432384 i_blocks 3365288 00000080:00200000:3.0:1787827173.135330:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.135333:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724432384, count: 1024 00000080:00200000:3.0:1787827173.135337:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421004 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.135339:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 0, original count 1024 00000080:00200000:3.0:1787827173.135341:0:27439:0:(file.c:2090:ll_file_io_generic()) oosfile: write ppos: 1724432384, bytes: 1024 00000080:00200000:3.0:1787827173.135343:0:27439:0:(file.c:1848:ll_io_set_mirror()) oosfile: desiginated mirror: 0 00000020:00200000:3.0:1787827173.135347:0:27439:0:(cl_io.c:218:cl_io_rw_init()) header@000000007788600a[0x110, 368557, [0x200000402:0x27:0x0] hash] 00000020:00200000:3.0:1787827173.135349:0:27439:0:(cl_io.c:218:cl_io_rw_init()) io range: 2 [1724432384, 1724433408) 0 0 00000080:00200000:3.0:1787827173.135351:0:27439:0:(vvp_io.c:1889:vvp_io_init()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1787827173.135357:0:27439:0:(file.c:2170:ll_file_io_generic()) Range lock [421004, 421004] 00020000:00200000:3.0:1787827173.135360:0:27439:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 411 chunk: [1724432384, 1724433408) 1724433408, 1024 00020000:00200000:3.0:1787827173.135362:0:27439:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1787827173.135367:0:27439:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [421004, 421004] 00000008:00010000:3.0:1787827173.135372:0:27439:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1787827173.135375:0:27439:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff9e35c7d81d30, osc lock ffff9e35c2ddcc30, flags 0x0 00010000:00010000:3.0:1787827173.135383:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.135391:0:27439:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (860405760 860409855) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.135399:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 20/0,2 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.135407:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 20/0,2 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.135414:0:27439:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 19/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.135421:0:27439:0:(vvp_io.c:1339:vvp_io_write_start()) oosfile: write [1724432384, 1724433408) append 0 append lockpos 0 00000080:00200000:3.0:1787827173.135426:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421004 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.135582:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724432384, len 1024, copied 1024 00000080:00200000:3.0:1787827173.135588:0:27439:0:(rw26.c:933:ll_write_end()) page@00000000d224284a[3 000000005ce0a915 2 1 00000000a23cbd2a 1 0 421004] 00000080:00200000:3.0:1787827173.135591:0:27439:0:(rw26.c:933:ll_write_end()) vmpage @0000000068c5f6e8 fffffc0001001 3:0 ffff9e34c3cb09f8 421004 lru 00000080:00200000:3.0:1787827173.135596:0:27439:0:(rw26.c:933:ll_write_end()) osc-page@00000000ce8669a8 210060: 1< 0 - - > 2< 860405760 0 0 0x0 0x0 | 000000004db7037e 0000000052729811 > 3< 0 0 > 4< 0 0 8 484040704 - | - - + - > 5< - - + - | 0 - | 358 - -> 00000080:00200000:3.0:1787827173.135599:0:27439:0:(rw26.c:933:ll_write_end()) end page@00000000d224284a 00000080:00200000:3.0:1787827173.135600:0:27439:0:(rw26.c:933:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1787827173.135603:0:27439:0:(vvp_io.c:1214:vvp_io_write_commit()) commit async pages: 1, from 0, to 1024 prio 0 00000008:00010000:3.0:1787827173.135614:0:27439:0:(osc_cache.c:565:osc_extent_merge()) ### extent: 000000001063fa34 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.135628:0:27439:0:(osc_cache.c:497:__osc_extent_remove()) ### extent: 000000001063fa34 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.135675:0:27439:0:(osc_cache.c:582:osc_extent_merge()) ### extent: 00000000df269729 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.135685:0:27439:0:(osc_cache.c:846:osc_extent_find()) ### extent: 00000000df269729 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.135700:0:27439:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff9e35d0e392f8, count 1, dirtied 1 00000080:00200000:3.0:1787827173.135703:0:27439:0:(vvp_io.c:1237:vvp_io_write_commit()) Committed 1 pages 1024 bytes, tot: 1024 00000080:00200000:3.0:1787827173.135706:0:27439:0:(vvp_io.c:1421:vvp_io_write_start()) oosfile: write bytes 0, result: 1024 00000020:00010000:3.0:1787827173.135714:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_release():245 00000020:00010000:3.0:1787827173.135717:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1787827173.135720:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c7d81d30 (ffff9e35da704cf8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1787827173.135725:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.135734:0:27439:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 17/0,0 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1787827173.135749:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_fini():72 00000020:00010000:3.0:1787827173.135752:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c7d81d30 (ffff9e35c614ce18/1) at cl_lock_fini():72 00000080:00200000:3.0:1787827173.135757:0:27439:0:(vvp_io.c:533:vvp_io_advance()) advancing 1024 bytes 00000080:00200000:3.0:1787827173.135760:0:27439:0:(file.c:2207:ll_file_io_generic()) Range unlock [421004, 421004] 00000080:00200000:3.0:1787827173.135766:0:27439:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787827173.135770:0:27439:0:(file.c:2240:ll_file_io_generic()) oosfile: 2 io complete with rc: 0, result: 1024, restart: 0 00000080:00200000:3.0:1787827173.135776:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724433408 i_blocks 3365288 00000080:00200000:3.0:1787827173.135779:0:27439:0:(file.c:2326:ll_file_io_generic()) iot: 2, result: 1024 00000080:00200008:3.0:1787827173.135788:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724433408, count: 1024 00000080:00200000:3.0:1787827173.135793:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421004 of 1024 to 1024 bytes 00000080:00200000:3.0:1787827173.135819:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724433408, len 1024, copied 1024 00000080:00200000:3.0:1787827173.135828:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724434432 i_blocks 3365288 00000080:00200000:3.0:1787827173.135831:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.135837:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724434432, count: 1024 00000080:00200000:3.0:1787827173.135841:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421004 of 2048 to 1024 bytes 00000080:00200000:3.0:1787827173.135844:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724434432, len 1024, copied 1024 00000080:00200000:3.0:1787827173.135850:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724435456 i_blocks 3365288 00000080:00200000:3.0:1787827173.135853:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.135858:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724435456, count: 1024 00000080:00200000:3.0:1787827173.135862:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421004 of 3072 to 1024 bytes 00000080:00200000:3.0:1787827173.135865:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724435456, len 1024, copied 1024 00000080:00200000:3.0:1787827173.135872:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724436480 i_blocks 3365288 00000080:00200000:3.0:1787827173.135875:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.135880:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724436480, count: 1024 00000080:00200000:3.0:1787827173.135884:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421005 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.135886:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 0, original count 1024 00000080:00200000:3.0:1787827173.135888:0:27439:0:(file.c:2090:ll_file_io_generic()) oosfile: write ppos: 1724436480, bytes: 1024 00000080:00200000:3.0:1787827173.135891:0:27439:0:(file.c:1848:ll_io_set_mirror()) oosfile: desiginated mirror: 0 00000020:00200000:3.0:1787827173.135895:0:27439:0:(cl_io.c:218:cl_io_rw_init()) header@000000007788600a[0x110, 368558, [0x200000402:0x27:0x0] hash] 00000020:00200000:3.0:1787827173.135897:0:27439:0:(cl_io.c:218:cl_io_rw_init()) io range: 2 [1724436480, 1724437504) 0 0 00000080:00200000:3.0:1787827173.135899:0:27439:0:(vvp_io.c:1889:vvp_io_init()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1787827173.135906:0:27439:0:(file.c:2170:ll_file_io_generic()) Range lock [421005, 421005] 00020000:00200000:3.0:1787827173.135909:0:27439:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 411 chunk: [1724436480, 1724437504) 1724437504, 1024 00020000:00200000:3.0:1787827173.135912:0:27439:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1787827173.135930:0:27439:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [421005, 421005] 00000008:00010000:3.0:1787827173.135938:0:27439:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1787827173.135941:0:27439:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff9e35c7d81d30, osc lock ffff9e35c2ddcc30, flags 0x0 00010000:00010000:3.0:1787827173.135950:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.135960:0:27439:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (860409856 860413951) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.135970:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 20/0,2 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.135979:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 20/0,2 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.135987:0:27439:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 19/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.135996:0:27439:0:(vvp_io.c:1339:vvp_io_write_start()) oosfile: write [1724436480, 1724437504) append 0 append lockpos 0 00000080:00200000:3.0:1787827173.136001:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421005 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.136021:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724436480, len 1024, copied 1024 00000080:00200000:3.0:1787827173.136026:0:27439:0:(rw26.c:933:ll_write_end()) page@00000000ec85463d[3 000000005ce0a915 2 1 00000000a23cbd2a 1 0 421005] 00000080:00200000:3.0:1787827173.136029:0:27439:0:(rw26.c:933:ll_write_end()) vmpage @00000000480e92af fffffc0001001 3:0 ffff9e34c3cb0ae0 421005 lru 00000080:00200000:3.0:1787827173.136035:0:27439:0:(rw26.c:933:ll_write_end()) osc-page@000000004496f3e1 210061: 1< 0 - - > 2< 860409856 0 0 0x0 0x0 | 000000004db7037e 0000000052729811 > 3< 0 0 > 4< 0 0 8 484036608 - | - - + - > 5< - - + - | 0 - | 359 - -> 00000080:00200000:3.0:1787827173.136037:0:27439:0:(rw26.c:933:ll_write_end()) end page@00000000ec85463d 00000080:00200000:3.0:1787827173.136039:0:27439:0:(rw26.c:933:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1787827173.136042:0:27439:0:(vvp_io.c:1214:vvp_io_write_commit()) commit async pages: 1, from 0, to 1024 prio 0 00000008:00010000:3.0:1787827173.136052:0:27439:0:(osc_cache.c:565:osc_extent_merge()) ### extent: 000000001063fa34 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.136067:0:27439:0:(osc_cache.c:497:__osc_extent_remove()) ### extent: 000000001063fa34 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.136074:0:27439:0:(osc_cache.c:582:osc_extent_merge()) ### extent: 00000000df269729 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.136082:0:27439:0:(osc_cache.c:846:osc_extent_find()) ### extent: 00000000df269729 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.136097:0:27439:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff9e35d0e392f8, count 1, dirtied 1 00000080:00200000:3.0:1787827173.136100:0:27439:0:(vvp_io.c:1237:vvp_io_write_commit()) Committed 1 pages 1024 bytes, tot: 1024 00000080:00200000:3.0:1787827173.136103:0:27439:0:(vvp_io.c:1421:vvp_io_write_start()) oosfile: write bytes 0, result: 1024 00000020:00010000:3.0:1787827173.136111:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_release():245 00000020:00010000:3.0:1787827173.136114:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1787827173.136117:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c7d81d30 (ffff9e35da704cf8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1787827173.136122:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.136131:0:27439:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 17/0,0 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1787827173.136142:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_fini():72 00000020:00010000:3.0:1787827173.136145:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c7d81d30 (ffff9e35c614ce18/1) at cl_lock_fini():72 00000080:00200000:3.0:1787827173.136150:0:27439:0:(vvp_io.c:533:vvp_io_advance()) advancing 1024 bytes 00000080:00200000:3.0:1787827173.136154:0:27439:0:(file.c:2207:ll_file_io_generic()) Range unlock [421005, 421005] 00000080:00200000:3.0:1787827173.136158:0:27439:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787827173.136161:0:27439:0:(file.c:2240:ll_file_io_generic()) oosfile: 2 io complete with rc: 0, result: 1024, restart: 0 00000080:00200000:3.0:1787827173.136166:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724437504 i_blocks 3365288 00000080:00200000:3.0:1787827173.136169:0:27439:0:(file.c:2326:ll_file_io_generic()) iot: 2, result: 1024 00000080:00200008:3.0:1787827173.136574:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724437504, count: 1024 00000080:00200000:3.0:1787827173.136581:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421005 of 1024 to 1024 bytes 00000080:00200000:3.0:1787827173.136585:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724437504, len 1024, copied 1024 00000080:00200000:3.0:1787827173.136593:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724438528 i_blocks 3365288 00000080:00200000:3.0:1787827173.136596:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.136602:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724438528, count: 1024 00000080:00200000:3.0:1787827173.136605:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421005 of 2048 to 1024 bytes 00000080:00200000:3.0:1787827173.136608:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724438528, len 1024, copied 1024 00000080:00200000:3.0:1787827173.136612:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724439552 i_blocks 3365288 00000080:00200000:3.0:1787827173.136615:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.136619:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724439552, count: 1024 00000080:00200000:3.0:1787827173.136623:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421005 of 3072 to 1024 bytes 00000080:00200000:3.0:1787827173.136626:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724439552, len 1024, copied 1024 00000080:00200000:3.0:1787827173.136630:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724440576 i_blocks 3365288 00000080:00200000:3.0:1787827173.136632:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.136636:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724440576, count: 1024 00000080:00200000:3.0:1787827173.136640:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421006 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.136642:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 0, original count 1024 00000080:00200000:3.0:1787827173.136644:0:27439:0:(file.c:2090:ll_file_io_generic()) oosfile: write ppos: 1724440576, bytes: 1024 00000080:00200000:3.0:1787827173.136647:0:27439:0:(file.c:1848:ll_io_set_mirror()) oosfile: desiginated mirror: 0 00000020:00200000:3.0:1787827173.136651:0:27439:0:(cl_io.c:218:cl_io_rw_init()) header@000000007788600a[0x110, 368559, [0x200000402:0x27:0x0] hash] 00000020:00200000:3.0:1787827173.136652:0:27439:0:(cl_io.c:218:cl_io_rw_init()) io range: 2 [1724440576, 1724441600) 0 0 00000080:00200000:3.0:1787827173.136655:0:27439:0:(vvp_io.c:1889:vvp_io_init()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1787827173.136661:0:27439:0:(file.c:2170:ll_file_io_generic()) Range lock [421006, 421006] 00020000:00200000:3.0:1787827173.136664:0:27439:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 411 chunk: [1724440576, 1724441600) 1724441600, 1024 00020000:00200000:3.0:1787827173.136667:0:27439:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1787827173.136673:0:27439:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [421006, 421006] 00000008:00010000:3.0:1787827173.136680:0:27439:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1787827173.136682:0:27439:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff9e35c7d81d30, osc lock ffff9e35c2ddcc30, flags 0x0 00010000:00010000:3.0:1787827173.136691:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.136700:0:27439:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (860413952 860418047) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.136709:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 20/0,2 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.136718:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 20/0,2 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.136725:0:27439:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 19/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.136733:0:27439:0:(vvp_io.c:1339:vvp_io_write_start()) oosfile: write [1724440576, 1724441600) append 0 append lockpos 0 00000080:00200000:3.0:1787827173.136738:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421006 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.136757:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724440576, len 1024, copied 1024 00000080:00200000:3.0:1787827173.136763:0:27439:0:(rw26.c:933:ll_write_end()) page@00000000b9503243[3 000000005ce0a915 2 1 00000000a23cbd2a 1 0 421006] 00000080:00200000:3.0:1787827173.136766:0:27439:0:(rw26.c:933:ll_write_end()) vmpage @00000000da526cae fffffc0001001 3:0 ffff9e34c3cb0828 421006 lru 00000080:00200000:3.0:1787827173.136772:0:27439:0:(rw26.c:933:ll_write_end()) osc-page@00000000d67e2c7e 210062: 1< 0 - - > 2< 860413952 0 0 0x0 0x0 | 000000004db7037e 0000000052729811 > 3< 0 0 > 4< 0 0 8 484032512 - | - - + - > 5< - - + - | 0 - | 360 - -> 00000080:00200000:3.0:1787827173.137027:0:27439:0:(rw26.c:933:ll_write_end()) end page@00000000b9503243 00000080:00200000:3.0:1787827173.137028:0:27439:0:(rw26.c:933:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1787827173.137031:0:27439:0:(vvp_io.c:1214:vvp_io_write_commit()) commit async pages: 1, from 0, to 1024 prio 0 00000008:00010000:3.0:1787827173.137042:0:27439:0:(osc_cache.c:565:osc_extent_merge()) ### extent: 000000001063fa34 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.137061:0:27439:0:(osc_cache.c:497:__osc_extent_remove()) ### extent: 000000001063fa34 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.137069:0:27439:0:(osc_cache.c:582:osc_extent_merge()) ### extent: 00000000df269729 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.137078:0:27439:0:(osc_cache.c:846:osc_extent_find()) ### extent: 00000000df269729 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.137547:0:27439:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff9e35d0e392f8, count 1, dirtied 1 00000080:00200000:3.0:1787827173.137552:0:27439:0:(vvp_io.c:1237:vvp_io_write_commit()) Committed 1 pages 1024 bytes, tot: 1024 00000080:00200000:3.0:1787827173.137555:0:27439:0:(vvp_io.c:1421:vvp_io_write_start()) oosfile: write bytes 0, result: 1024 00000020:00010000:3.0:1787827173.137564:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_release():245 00000020:00010000:3.0:1787827173.137567:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1787827173.137570:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c7d81d30 (ffff9e35da704cf8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1787827173.137576:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.137586:0:27439:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 17/0,0 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1787827173.137597:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_fini():72 00000020:00010000:3.0:1787827173.137599:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c7d81d30 (ffff9e35c614ce18/1) at cl_lock_fini():72 00000080:00200000:3.0:1787827173.137604:0:27439:0:(vvp_io.c:533:vvp_io_advance()) advancing 1024 bytes 00000080:00200000:3.0:1787827173.137607:0:27439:0:(file.c:2207:ll_file_io_generic()) Range unlock [421006, 421006] 00000080:00200000:3.0:1787827173.137613:0:27439:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787827173.137616:0:27439:0:(file.c:2240:ll_file_io_generic()) oosfile: 2 io complete with rc: 0, result: 1024, restart: 0 00000080:00200000:3.0:1787827173.137622:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724441600 i_blocks 3365288 00000080:00200000:3.0:1787827173.137625:0:27439:0:(file.c:2326:ll_file_io_generic()) iot: 2, result: 1024 00000080:00200008:3.0:1787827173.137635:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724441600, count: 1024 00000080:00200000:3.0:1787827173.137641:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421006 of 1024 to 1024 bytes 00000080:00200000:3.0:1787827173.137646:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724441600, len 1024, copied 1024 00000080:00200000:3.0:1787827173.137652:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724442624 i_blocks 3365288 00000080:00200000:3.0:1787827173.137655:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.137659:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724442624, count: 1024 00000080:00200000:3.0:1787827173.137663:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421006 of 2048 to 1024 bytes 00000080:00200000:3.0:1787827173.137665:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724442624, len 1024, copied 1024 00000080:00200000:3.0:1787827173.137668:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724443648 i_blocks 3365288 00000080:00200000:3.0:1787827173.137671:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.137674:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724443648, count: 1024 00000080:00200000:3.0:1787827173.137678:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421006 of 3072 to 1024 bytes 00000080:00200000:3.0:1787827173.137680:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724443648, len 1024, copied 1024 00000080:00200000:3.0:1787827173.137684:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724444672 i_blocks 3365288 00000080:00200000:3.0:1787827173.137687:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.137690:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724444672, count: 1024 00000080:00200000:3.0:1787827173.137694:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421007 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.137696:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 0, original count 1024 00000080:00200000:3.0:1787827173.137698:0:27439:0:(file.c:2090:ll_file_io_generic()) oosfile: write ppos: 1724444672, bytes: 1024 00000080:00200000:3.0:1787827173.137701:0:27439:0:(file.c:1848:ll_io_set_mirror()) oosfile: desiginated mirror: 0 00000020:00200000:3.0:1787827173.137909:0:27439:0:(cl_io.c:218:cl_io_rw_init()) header@000000007788600a[0x110, 368560, [0x200000402:0x27:0x0] hash] 00000020:00200000:3.0:1787827173.137912:0:27439:0:(cl_io.c:218:cl_io_rw_init()) io range: 2 [1724444672, 1724445696) 0 0 00000080:00200000:3.0:1787827173.137915:0:27439:0:(vvp_io.c:1889:vvp_io_init()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1787827173.137935:0:27439:0:(file.c:2170:ll_file_io_generic()) Range lock [421007, 421007] 00020000:00200000:3.0:1787827173.137939:0:27439:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 411 chunk: [1724444672, 1724445696) 1724445696, 1024 00020000:00200000:3.0:1787827173.137943:0:27439:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1787827173.137952:0:27439:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [421007, 421007] 00000008:00010000:3.0:1787827173.137959:0:27439:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1787827173.137962:0:27439:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff9e35c7d81d30, osc lock ffff9e35c2ddcc30, flags 0x0 00010000:00010000:3.0:1787827173.137979:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.137989:0:27439:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (860418048 860422143) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.137999:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 20/0,2 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.138007:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 20/0,2 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.138015:0:27439:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 19/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.138023:0:27439:0:(vvp_io.c:1339:vvp_io_write_start()) oosfile: write [1724444672, 1724445696) append 0 append lockpos 0 00000080:00200000:3.0:1787827173.138030:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421007 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.138053:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724444672, len 1024, copied 1024 00000080:00200000:3.0:1787827173.138059:0:27439:0:(rw26.c:933:ll_write_end()) page@00000000c07233e1[3 000000005ce0a915 2 1 00000000a23cbd2a 1 0 421007] 00000080:00200000:3.0:1787827173.138062:0:27439:0:(rw26.c:933:ll_write_end()) vmpage @000000000572b693 fffffc0001001 3:0 ffff9e34c3cb0570 421007 lru 00000080:00200000:3.0:1787827173.138068:0:27439:0:(rw26.c:933:ll_write_end()) osc-page@00000000dc01232e 210063: 1< 0 - - > 2< 860418048 0 0 0x0 0x0 | 000000004db7037e 0000000052729811 > 3< 0 0 > 4< 0 0 8 484028416 - | - - + - > 5< - - + - | 0 - | 361 - -> 00000080:00200000:3.0:1787827173.138071:0:27439:0:(rw26.c:933:ll_write_end()) end page@00000000c07233e1 00000080:00200000:3.0:1787827173.138072:0:27439:0:(rw26.c:933:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1787827173.138074:0:27439:0:(vvp_io.c:1214:vvp_io_write_commit()) commit async pages: 1, from 0, to 1024 prio 0 00000008:00010000:3.0:1787827173.138083:0:27439:0:(osc_cache.c:565:osc_extent_merge()) ### extent: 000000001063fa34 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.138097:0:27439:0:(osc_cache.c:497:__osc_extent_remove()) ### extent: 000000001063fa34 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.138112:0:27439:0:(osc_cache.c:582:osc_extent_merge()) ### extent: 00000000df269729 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.138374:0:27439:0:(osc_cache.c:846:osc_extent_find()) ### extent: 00000000df269729 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.138390:0:27439:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff9e35d0e392f8, count 1, dirtied 1 00000080:00200000:3.0:1787827173.138393:0:27439:0:(vvp_io.c:1237:vvp_io_write_commit()) Committed 1 pages 1024 bytes, tot: 1024 00000080:00200000:3.0:1787827173.138396:0:27439:0:(vvp_io.c:1421:vvp_io_write_start()) oosfile: write bytes 0, result: 1024 00000020:00010000:3.0:1787827173.138403:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_release():245 00000020:00010000:3.0:1787827173.138406:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1787827173.138409:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c7d81d30 (ffff9e35da704cf8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1787827173.138413:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.138422:0:27439:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 17/0,0 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1787827173.138432:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_fini():72 00000020:00010000:3.0:1787827173.138435:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c7d81d30 (ffff9e35c614ce18/1) at cl_lock_fini():72 00000080:00200000:3.0:1787827173.138440:0:27439:0:(vvp_io.c:533:vvp_io_advance()) advancing 1024 bytes 00000080:00200000:3.0:1787827173.138444:0:27439:0:(file.c:2207:ll_file_io_generic()) Range unlock [421007, 421007] 00000080:00200000:3.0:1787827173.138448:0:27439:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787827173.138452:0:27439:0:(file.c:2240:ll_file_io_generic()) oosfile: 2 io complete with rc: 0, result: 1024, restart: 0 00000080:00200000:3.0:1787827173.138458:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724445696 i_blocks 3365288 00000080:00200000:3.0:1787827173.138461:0:27439:0:(file.c:2326:ll_file_io_generic()) iot: 2, result: 1024 00000080:00200008:3.0:1787827173.138472:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724445696, count: 1024 00000080:00200000:3.0:1787827173.138477:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421007 of 1024 to 1024 bytes 00000080:00200000:3.0:1787827173.138481:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724445696, len 1024, copied 1024 00000080:00200000:3.0:1787827173.138487:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724446720 i_blocks 3365288 00000080:00200000:3.0:1787827173.138490:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.138495:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724446720, count: 1024 00000080:00200000:3.0:1787827173.138857:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421007 of 2048 to 1024 bytes 00000080:00200000:3.0:1787827173.138862:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724446720, len 1024, copied 1024 00000080:00200000:3.0:1787827173.138869:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724447744 i_blocks 3365288 00000080:00200000:3.0:1787827173.138873:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.138880:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724447744, count: 1024 00000080:00200000:3.0:1787827173.138885:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421007 of 3072 to 1024 bytes 00000080:00200000:3.0:1787827173.138887:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724447744, len 1024, copied 1024 00000080:00200000:3.0:1787827173.138892:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724448768 i_blocks 3365288 00000080:00200000:3.0:1787827173.138894:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.138899:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724448768, count: 1024 00000080:00200000:3.0:1787827173.138903:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421008 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.138905:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 0, original count 1024 00000080:00200000:3.0:1787827173.138908:0:27439:0:(file.c:2090:ll_file_io_generic()) oosfile: write ppos: 1724448768, bytes: 1024 00000080:00200000:3.0:1787827173.138910:0:27439:0:(file.c:1848:ll_io_set_mirror()) oosfile: desiginated mirror: 0 00000020:00200000:3.0:1787827173.138915:0:27439:0:(cl_io.c:218:cl_io_rw_init()) header@000000007788600a[0x110, 368561, [0x200000402:0x27:0x0] hash] 00000020:00200000:3.0:1787827173.138917:0:27439:0:(cl_io.c:218:cl_io_rw_init()) io range: 2 [1724448768, 1724449792) 0 0 00000080:00200000:3.0:1787827173.138936:0:27439:0:(vvp_io.c:1889:vvp_io_init()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1787827173.138944:0:27439:0:(file.c:2170:ll_file_io_generic()) Range lock [421008, 421008] 00020000:00200000:3.0:1787827173.138947:0:27439:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 411 chunk: [1724448768, 1724449792) 1724449792, 1024 00020000:00200000:3.0:1787827173.138951:0:27439:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1787827173.138958:0:27439:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [421008, 421008] 00000008:00010000:3.0:1787827173.138964:0:27439:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1787827173.138967:0:27439:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff9e35c7d81d30, osc lock ffff9e35c2ddcc30, flags 0x0 00010000:00010000:3.0:1787827173.138976:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.138986:0:27439:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (860422144 860426239) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.138996:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 20/0,2 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.139322:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 20/0,2 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.139333:0:27439:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 19/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.139343:0:27439:0:(vvp_io.c:1339:vvp_io_write_start()) oosfile: write [1724448768, 1724449792) append 0 append lockpos 0 00000080:00200000:3.0:1787827173.139351:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421008 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.139375:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724448768, len 1024, copied 1024 00000080:00200000:3.0:1787827173.139381:0:27439:0:(rw26.c:933:ll_write_end()) page@0000000087feb864[3 000000005ce0a915 2 1 00000000a23cbd2a 1 0 421008] 00000080:00200000:3.0:1787827173.139384:0:27439:0:(rw26.c:933:ll_write_end()) vmpage @0000000067e31af6 fffffc0001001 3:0 ffff9e34c3cb0e80 421008 lru 00000080:00200000:3.0:1787827173.139391:0:27439:0:(rw26.c:933:ll_write_end()) osc-page@00000000dfb67ffd 210064: 1< 0 - - > 2< 860422144 0 0 0x0 0x0 | 000000004db7037e 0000000052729811 > 3< 0 0 > 4< 0 0 8 484024320 - | - - + - > 5< - - + - | 0 - | 362 - -> 00000080:00200000:3.0:1787827173.139394:0:27439:0:(rw26.c:933:ll_write_end()) end page@0000000087feb864 00000080:00200000:3.0:1787827173.139395:0:27439:0:(rw26.c:933:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1787827173.139397:0:27439:0:(vvp_io.c:1214:vvp_io_write_commit()) commit async pages: 1, from 0, to 1024 prio 0 00000008:00010000:3.0:1787827173.139408:0:27439:0:(osc_cache.c:565:osc_extent_merge()) ### extent: 000000001063fa34 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.139424:0:27439:0:(osc_cache.c:497:__osc_extent_remove()) ### extent: 000000001063fa34 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.139432:0:27439:0:(osc_cache.c:582:osc_extent_merge()) ### extent: 00000000df269729 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.139441:0:27439:0:(osc_cache.c:846:osc_extent_find()) ### extent: 00000000df269729 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.139454:0:27439:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff9e35d0e392f8, count 1, dirtied 1 00000080:00200000:3.0:1787827173.139458:0:27439:0:(vvp_io.c:1237:vvp_io_write_commit()) Committed 1 pages 1024 bytes, tot: 1024 00000080:00200000:3.0:1787827173.139461:0:27439:0:(vvp_io.c:1421:vvp_io_write_start()) oosfile: write bytes 0, result: 1024 00000020:00010000:3.0:1787827173.139468:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_release():245 00000020:00010000:3.0:1787827173.139471:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1787827173.139830:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c7d81d30 (ffff9e35da704cf8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1787827173.139839:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.139851:0:27439:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 17/0,0 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1787827173.139864:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_fini():72 00000020:00010000:3.0:1787827173.139867:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c7d81d30 (ffff9e35c614ce18/1) at cl_lock_fini():72 00000080:00200000:3.0:1787827173.139874:0:27439:0:(vvp_io.c:533:vvp_io_advance()) advancing 1024 bytes 00000080:00200000:3.0:1787827173.139878:0:27439:0:(file.c:2207:ll_file_io_generic()) Range unlock [421008, 421008] 00000080:00200000:3.0:1787827173.139883:0:27439:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787827173.139887:0:27439:0:(file.c:2240:ll_file_io_generic()) oosfile: 2 io complete with rc: 0, result: 1024, restart: 0 00000080:00200000:3.0:1787827173.139893:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724449792 i_blocks 3365288 00000080:00200000:3.0:1787827173.139909:0:27439:0:(file.c:2326:ll_file_io_generic()) iot: 2, result: 1024 00000080:00200008:3.0:1787827173.139930:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724449792, count: 1024 00000080:00200000:3.0:1787827173.139937:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421008 of 1024 to 1024 bytes 00000080:00200000:3.0:1787827173.139941:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724449792, len 1024, copied 1024 00000080:00200000:3.0:1787827173.139947:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724450816 i_blocks 3365288 00000080:00200000:3.0:1787827173.139950:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.139959:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724450816, count: 1024 00000080:00200000:3.0:1787827173.139963:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421008 of 2048 to 1024 bytes 00000080:00200000:3.0:1787827173.139965:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724450816, len 1024, copied 1024 00000080:00200000:3.0:1787827173.139969:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724451840 i_blocks 3365288 00000080:00200000:3.0:1787827173.139971:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.139974:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724451840, count: 1024 00000080:00200000:3.0:1787827173.139977:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421008 of 3072 to 1024 bytes 00000080:00200000:3.0:1787827173.139979:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724451840, len 1024, copied 1024 00000080:00200000:3.0:1787827173.139983:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724452864 i_blocks 3365288 00000080:00200000:3.0:1787827173.139985:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.139988:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724452864, count: 1024 00000080:00200000:3.0:1787827173.139991:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421009 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.139993:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 0, original count 1024 00000080:00200000:3.0:1787827173.139995:0:27439:0:(file.c:2090:ll_file_io_generic()) oosfile: write ppos: 1724452864, bytes: 1024 00000080:00200000:3.0:1787827173.139997:0:27439:0:(file.c:1848:ll_io_set_mirror()) oosfile: desiginated mirror: 0 00000020:00200000:3.0:1787827173.140001:0:27439:0:(cl_io.c:218:cl_io_rw_init()) header@000000007788600a[0x110, 368562, [0x200000402:0x27:0x0] hash] 00000020:00200000:3.0:1787827173.140003:0:27439:0:(cl_io.c:218:cl_io_rw_init()) io range: 2 [1724452864, 1724453888) 0 0 00000080:00200000:3.0:1787827173.140006:0:27439:0:(vvp_io.c:1889:vvp_io_init()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1787827173.140013:0:27439:0:(file.c:2170:ll_file_io_generic()) Range lock [421009, 421009] 00020000:00200000:3.0:1787827173.140017:0:27439:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 411 chunk: [1724452864, 1724453888) 1724453888, 1024 00020000:00200000:3.0:1787827173.140468:0:27439:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1787827173.140480:0:27439:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [421009, 421009] 00000008:00010000:3.0:1787827173.140489:0:27439:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1787827173.140492:0:27439:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff9e35c7d81d30, osc lock ffff9e35c2ddcc30, flags 0x0 00010000:00010000:3.0:1787827173.140504:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.140515:0:27439:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (860426240 860430335) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.140526:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 20/0,2 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.140533:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 20/0,2 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.140539:0:27439:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 19/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.140558:0:27439:0:(vvp_io.c:1339:vvp_io_write_start()) oosfile: write [1724452864, 1724453888) append 0 append lockpos 0 00000080:00200000:3.0:1787827173.140565:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421009 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.140589:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724452864, len 1024, copied 1024 00000080:00200000:3.0:1787827173.140595:0:27439:0:(rw26.c:933:ll_write_end()) page@0000000013624b71[3 000000005ce0a915 2 1 00000000a23cbd2a 1 0 421009] 00000080:00200000:3.0:1787827173.140599:0:27439:0:(rw26.c:933:ll_write_end()) vmpage @00000000a5ff96ca fffffc0001001 3:0 ffff9e34c3cb0740 421009 lru 00000080:00200000:3.0:1787827173.140605:0:27439:0:(rw26.c:933:ll_write_end()) osc-page@00000000eeac584f 210065: 1< 0 - - > 2< 860426240 0 0 0x0 0x0 | 000000004db7037e 0000000052729811 > 3< 0 0 > 4< 0 0 8 484020224 - | - - + - > 5< - - + - | 0 - | 363 - -> 00000080:00200000:3.0:1787827173.140607:0:27439:0:(rw26.c:933:ll_write_end()) end page@0000000013624b71 00000080:00200000:3.0:1787827173.140608:0:27439:0:(rw26.c:933:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1787827173.140611:0:27439:0:(vvp_io.c:1214:vvp_io_write_commit()) commit async pages: 1, from 0, to 1024 prio 0 00000008:00010000:3.0:1787827173.140622:0:27439:0:(osc_cache.c:565:osc_extent_merge()) ### extent: 000000001063fa34 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.140636:0:27439:0:(osc_cache.c:497:__osc_extent_remove()) ### extent: 000000001063fa34 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.140889:0:27439:0:(osc_cache.c:582:osc_extent_merge()) ### extent: 00000000df269729 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.140899:0:27439:0:(osc_cache.c:846:osc_extent_find()) ### extent: 00000000df269729 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.140915:0:27439:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff9e35d0e392f8, count 1, dirtied 1 00000080:00200000:3.0:1787827173.140931:0:27439:0:(vvp_io.c:1237:vvp_io_write_commit()) Committed 1 pages 1024 bytes, tot: 1024 00000080:00200000:3.0:1787827173.140935:0:27439:0:(vvp_io.c:1421:vvp_io_write_start()) oosfile: write bytes 0, result: 1024 00000020:00010000:3.0:1787827173.140943:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_release():245 00000020:00010000:3.0:1787827173.140947:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1787827173.140949:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c7d81d30 (ffff9e35da704cf8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1787827173.140954:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.140962:0:27439:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 17/0,0 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1787827173.140971:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_fini():72 00000020:00010000:3.0:1787827173.140974:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c7d81d30 (ffff9e35c614ce18/1) at cl_lock_fini():72 00000080:00200000:3.0:1787827173.140979:0:27439:0:(vvp_io.c:533:vvp_io_advance()) advancing 1024 bytes 00000080:00200000:3.0:1787827173.140982:0:27439:0:(file.c:2207:ll_file_io_generic()) Range unlock [421009, 421009] 00000080:00200000:3.0:1787827173.140987:0:27439:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787827173.140990:0:27439:0:(file.c:2240:ll_file_io_generic()) oosfile: 2 io complete with rc: 0, result: 1024, restart: 0 00000080:00200000:3.0:1787827173.140996:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724453888 i_blocks 3365288 00000080:00200000:3.0:1787827173.141000:0:27439:0:(file.c:2326:ll_file_io_generic()) iot: 2, result: 1024 00000080:00200008:3.0:1787827173.141011:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724453888, count: 1024 00000080:00200000:3.0:1787827173.141016:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421009 of 1024 to 1024 bytes 00000080:00200000:3.0:1787827173.141019:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724453888, len 1024, copied 1024 00000080:00200000:3.0:1787827173.141025:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724454912 i_blocks 3365288 00000080:00200000:3.0:1787827173.141028:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.141032:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724454912, count: 1024 00000080:00200000:3.0:1787827173.141036:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421009 of 2048 to 1024 bytes 00000080:00200000:3.0:1787827173.141039:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724454912, len 1024, copied 1024 00000080:00200000:3.0:1787827173.141042:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724455936 i_blocks 3365288 00000080:00200000:3.0:1787827173.141045:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.141049:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724455936, count: 1024 00000080:00200000:3.0:1787827173.141052:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421009 of 3072 to 1024 bytes 00000080:00200000:3.0:1787827173.141054:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724455936, len 1024, copied 1024 00000080:00200000:3.0:1787827173.141296:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724456960 i_blocks 3365288 00000080:00200000:3.0:1787827173.141300:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.141308:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724456960, count: 1024 00000080:00200000:3.0:1787827173.141314:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421010 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.141318:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 0, original count 1024 00000080:00200000:3.0:1787827173.141321:0:27439:0:(file.c:2090:ll_file_io_generic()) oosfile: write ppos: 1724456960, bytes: 1024 00000080:00200000:3.0:1787827173.141324:0:27439:0:(file.c:1848:ll_io_set_mirror()) oosfile: desiginated mirror: 0 00000020:00200000:3.0:1787827173.141329:0:27439:0:(cl_io.c:218:cl_io_rw_init()) header@000000007788600a[0x110, 368563, [0x200000402:0x27:0x0] hash] 00000020:00200000:3.0:1787827173.141332:0:27439:0:(cl_io.c:218:cl_io_rw_init()) io range: 2 [1724456960, 1724457984) 0 0 00000080:00200000:3.0:1787827173.141335:0:27439:0:(vvp_io.c:1889:vvp_io_init()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1787827173.141344:0:27439:0:(file.c:2170:ll_file_io_generic()) Range lock [421010, 421010] 00020000:00200000:3.0:1787827173.141349:0:27439:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 411 chunk: [1724456960, 1724457984) 1724457984, 1024 00020000:00200000:3.0:1787827173.141352:0:27439:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1787827173.141360:0:27439:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [421010, 421010] 00000008:00010000:3.0:1787827173.141367:0:27439:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1787827173.141370:0:27439:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff9e35c7d81d30, osc lock ffff9e35c2ddcc30, flags 0x0 00010000:00010000:3.0:1787827173.141380:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.141389:0:27439:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (860430336 860434431) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.141397:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 20/0,2 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.141405:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 20/0,2 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.141412:0:27439:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 19/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.141421:0:27439:0:(vvp_io.c:1339:vvp_io_write_start()) oosfile: write [1724456960, 1724457984) append 0 append lockpos 0 00000080:00200000:3.0:1787827173.141427:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421010 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.141669:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724456960, len 1024, copied 1024 00000080:00200000:3.0:1787827173.141674:0:27439:0:(rw26.c:933:ll_write_end()) page@00000000f0b09c25[3 000000005ce0a915 2 1 00000000a23cbd2a 1 0 421010] 00000080:00200000:3.0:1787827173.141677:0:27439:0:(rw26.c:933:ll_write_end()) vmpage @00000000ec67bf15 fffffc0001001 3:0 ffff9e34c3cb0910 421010 lru 00000080:00200000:3.0:1787827173.141681:0:27439:0:(rw26.c:933:ll_write_end()) osc-page@00000000a8da93c8 210066: 1< 0 - - > 2< 860430336 0 0 0x0 0x0 | 000000004db7037e 0000000052729811 > 3< 0 0 > 4< 0 0 8 484016128 - | - - + - > 5< - - + - | 0 - | 364 - -> 00000080:00200000:3.0:1787827173.141682:0:27439:0:(rw26.c:933:ll_write_end()) end page@00000000f0b09c25 00000080:00200000:3.0:1787827173.141683:0:27439:0:(rw26.c:933:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1787827173.141685:0:27439:0:(vvp_io.c:1214:vvp_io_write_commit()) commit async pages: 1, from 0, to 1024 prio 0 00000008:00010000:3.0:1787827173.141692:0:27439:0:(osc_cache.c:565:osc_extent_merge()) ### extent: 000000001063fa34 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.141702:0:27439:0:(osc_cache.c:497:__osc_extent_remove()) ### extent: 000000001063fa34 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.141707:0:27439:0:(osc_cache.c:582:osc_extent_merge()) ### extent: 00000000df269729 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.141711:0:27439:0:(osc_cache.c:846:osc_extent_find()) ### extent: 00000000df269729 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.141718:0:27439:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff9e35d0e392f8, count 1, dirtied 1 00000080:00200000:3.0:1787827173.141720:0:27439:0:(vvp_io.c:1237:vvp_io_write_commit()) Committed 1 pages 1024 bytes, tot: 1024 00000080:00200000:3.0:1787827173.141722:0:27439:0:(vvp_io.c:1421:vvp_io_write_start()) oosfile: write bytes 0, result: 1024 00000020:00010000:3.0:1787827173.141727:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_release():245 00000020:00010000:3.0:1787827173.141729:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1787827173.141730:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c7d81d30 (ffff9e35da704cf8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1787827173.141733:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.141739:0:27439:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 17/0,0 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1787827173.141745:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_fini():72 00000020:00010000:3.0:1787827173.141748:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c7d81d30 (ffff9e35c614ce18/1) at cl_lock_fini():72 00000080:00200000:3.0:1787827173.141752:0:27439:0:(vvp_io.c:533:vvp_io_advance()) advancing 1024 bytes 00000080:00200000:3.0:1787827173.141755:0:27439:0:(file.c:2207:ll_file_io_generic()) Range unlock [421010, 421010] 00000080:00200000:3.0:1787827173.141759:0:27439:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787827173.141761:0:27439:0:(file.c:2240:ll_file_io_generic()) oosfile: 2 io complete with rc: 0, result: 1024, restart: 0 00000080:00200000:3.0:1787827173.141765:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724457984 i_blocks 3365288 00000080:00200000:3.0:1787827173.141767:0:27439:0:(file.c:2326:ll_file_io_generic()) iot: 2, result: 1024 00000080:00200008:3.0:1787827173.141774:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724457984, count: 1024 00000080:00200000:3.0:1787827173.141777:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421010 of 1024 to 1024 bytes 00000080:00200000:3.0:1787827173.141779:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724457984, len 1024, copied 1024 00000080:00200000:3.0:1787827173.141783:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724459008 i_blocks 3365288 00000080:00200000:3.0:1787827173.141982:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.141991:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724459008, count: 1024 00000080:00200000:3.0:1787827173.141997:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421010 of 2048 to 1024 bytes 00000080:00200000:3.0:1787827173.142000:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724459008, len 1024, copied 1024 00000080:00200000:3.0:1787827173.142188:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724460032 i_blocks 3365288 00000080:00200000:3.0:1787827173.142194:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.142200:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724460032, count: 1024 00000080:00200000:3.0:1787827173.142205:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421010 of 3072 to 1024 bytes 00000080:00200000:3.0:1787827173.142209:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724460032, len 1024, copied 1024 00000080:00200000:3.0:1787827173.142225:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724461056 i_blocks 3365288 00000080:00200000:3.0:1787827173.142314:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.142329:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724461056, count: 1024 00000080:00200000:3.0:1787827173.142334:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421011 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.142337:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 0, original count 1024 00000080:00200000:3.0:1787827173.142339:0:27439:0:(file.c:2090:ll_file_io_generic()) oosfile: write ppos: 1724461056, bytes: 1024 00000080:00200000:3.0:1787827173.142342:0:27439:0:(file.c:1848:ll_io_set_mirror()) oosfile: desiginated mirror: 0 00000020:00200000:3.0:1787827173.142346:0:27439:0:(cl_io.c:218:cl_io_rw_init()) header@000000007788600a[0x110, 368564, [0x200000402:0x27:0x0] hash] 00000020:00200000:3.0:1787827173.142348:0:27439:0:(cl_io.c:218:cl_io_rw_init()) io range: 2 [1724461056, 1724462080) 0 0 00000080:00200000:3.0:1787827173.142351:0:27439:0:(vvp_io.c:1889:vvp_io_init()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1787827173.142358:0:27439:0:(file.c:2170:ll_file_io_generic()) Range lock [421011, 421011] 00020000:00200000:3.0:1787827173.142362:0:27439:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 411 chunk: [1724461056, 1724462080) 1724462080, 1024 00020000:00200000:3.0:1787827173.142375:0:27439:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1787827173.142383:0:27439:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [421011, 421011] 00000008:00010000:3.0:1787827173.142390:0:27439:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1787827173.142393:0:27439:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff9e35c7d81d30, osc lock ffff9e35c2ddcc30, flags 0x0 00010000:00010000:3.0:1787827173.142403:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.142414:0:27439:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (860434432 860438527) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.142423:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 20/0,2 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.142432:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 20/0,2 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.142439:0:27439:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 19/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.142503:0:27439:0:(vvp_io.c:1339:vvp_io_write_start()) oosfile: write [1724461056, 1724462080) append 0 append lockpos 0 00000080:00200000:3.0:1787827173.142511:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421011 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.142538:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724461056, len 1024, copied 1024 00000080:00200000:3.0:1787827173.142543:0:27439:0:(rw26.c:933:ll_write_end()) page@0000000034021e7e[3 000000005ce0a915 2 1 00000000a23cbd2a 1 0 421011] 00000080:00200000:3.0:1787827173.142546:0:27439:0:(rw26.c:933:ll_write_end()) vmpage @00000000c1bdfb6d fffffc0001021 2:0 ffff9e34c3cb00e8 421011 lru 00000080:00200000:3.0:1787827173.142552:0:27439:0:(rw26.c:933:ll_write_end()) osc-page@0000000066ee099a 210067: 1< 0 - - > 2< 860434432 0 0 0x0 0x0 | 000000004db7037e 0000000052729811 > 3< 0 0 > 4< 0 0 8 484012032 - | - - + - > 5< - - + - | 0 - | 365 - -> 00000080:00200000:3.0:1787827173.142554:0:27439:0:(rw26.c:933:ll_write_end()) end page@0000000034021e7e 00000080:00200000:3.0:1787827173.142555:0:27439:0:(rw26.c:933:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1787827173.142557:0:27439:0:(vvp_io.c:1214:vvp_io_write_commit()) commit async pages: 1, from 0, to 1024 prio 0 00000008:00010000:3.0:1787827173.142625:0:27439:0:(osc_cache.c:565:osc_extent_merge()) ### extent: 000000001063fa34 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.142640:0:27439:0:(osc_cache.c:497:__osc_extent_remove()) ### extent: 000000001063fa34 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.142648:0:27439:0:(osc_cache.c:582:osc_extent_merge()) ### extent: 00000000df269729 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.142656:0:27439:0:(osc_cache.c:846:osc_extent_find()) ### extent: 00000000df269729 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.142668:0:27439:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff9e35d0e392f8, count 1, dirtied 1 00000080:00200000:3.0:1787827173.142672:0:27439:0:(vvp_io.c:1237:vvp_io_write_commit()) Committed 1 pages 1024 bytes, tot: 1024 00000080:00200000:3.0:1787827173.142675:0:27439:0:(vvp_io.c:1421:vvp_io_write_start()) oosfile: write bytes 0, result: 1024 00000020:00010000:3.0:1787827173.142683:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_release():245 00000020:00010000:3.0:1787827173.142687:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1787827173.142690:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c7d81d30 (ffff9e35da704cf8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1787827173.142703:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.142712:0:27439:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 17/0,0 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1787827173.142724:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_fini():72 00000020:00010000:3.0:1787827173.142727:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c7d81d30 (ffff9e35c614ce18/1) at cl_lock_fini():72 00000080:00200000:3.0:1787827173.142733:0:27439:0:(vvp_io.c:533:vvp_io_advance()) advancing 1024 bytes 00000080:00200000:3.0:1787827173.142738:0:27439:0:(file.c:2207:ll_file_io_generic()) Range unlock [421011, 421011] 00000080:00200000:3.0:1787827173.142743:0:27439:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787827173.142746:0:27439:0:(file.c:2240:ll_file_io_generic()) oosfile: 2 io complete with rc: 0, result: 1024, restart: 0 00000080:00200000:3.0:1787827173.142751:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724462080 i_blocks 3365288 00000080:00200000:3.0:1787827173.142754:0:27439:0:(file.c:2326:ll_file_io_generic()) iot: 2, result: 1024 00000080:00200008:3.0:1787827173.142762:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724462080, count: 1024 00000080:00200000:3.0:1787827173.142767:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421011 of 1024 to 1024 bytes 00000080:00200000:3.0:1787827173.142770:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724462080, len 1024, copied 1024 00000080:00200000:3.0:1787827173.142775:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724463104 i_blocks 3365288 00000080:00200000:3.0:1787827173.142778:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.142781:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724463104, count: 1024 00000080:00200000:3.0:1787827173.142784:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421011 of 2048 to 1024 bytes 00000080:00200000:3.0:1787827173.142786:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724463104, len 1024, copied 1024 00000080:00200000:3.0:1787827173.142790:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724464128 i_blocks 3365288 00000080:00200000:3.0:1787827173.142792:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.142821:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724464128, count: 1024 00000080:00200000:3.0:1787827173.142826:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421011 of 3072 to 1024 bytes 00000080:00200000:3.0:1787827173.142829:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724464128, len 1024, copied 1024 00000080:00200000:3.0:1787827173.142833:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724465152 i_blocks 3365288 00000080:00200000:3.0:1787827173.142836:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.142840:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724465152, count: 1024 00000080:00200000:3.0:1787827173.142844:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421012 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.142846:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 0, original count 1024 00000080:00200000:3.0:1787827173.142848:0:27439:0:(file.c:2090:ll_file_io_generic()) oosfile: write ppos: 1724465152, bytes: 1024 00000080:00200000:3.0:1787827173.142850:0:27439:0:(file.c:1848:ll_io_set_mirror()) oosfile: desiginated mirror: 0 00000020:00200000:3.0:1787827173.142854:0:27439:0:(cl_io.c:218:cl_io_rw_init()) header@000000007788600a[0x110, 368565, [0x200000402:0x27:0x0] hash] 00000020:00200000:3.0:1787827173.142856:0:27439:0:(cl_io.c:218:cl_io_rw_init()) io range: 2 [1724465152, 1724466176) 0 0 00000080:00200000:3.0:1787827173.142858:0:27439:0:(vvp_io.c:1889:vvp_io_init()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1787827173.142864:0:27439:0:(file.c:2170:ll_file_io_generic()) Range lock [421012, 421012] 00020000:00200000:3.0:1787827173.142867:0:27439:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 411 chunk: [1724465152, 1724466176) 1724466176, 1024 00020000:00200000:3.0:1787827173.142870:0:27439:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1787827173.142884:0:27439:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [421012, 421012] 00000008:00010000:3.0:1787827173.142893:0:27439:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1787827173.142896:0:27439:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff9e35c7d81d30, osc lock ffff9e35c2ddcc30, flags 0x0 00010000:00010000:3.0:1787827173.142904:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.142913:0:27439:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (860438528 860442623) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.142934:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 20/0,2 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.142941:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 20/0,2 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.142948:0:27439:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 19/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.142955:0:27439:0:(vvp_io.c:1339:vvp_io_write_start()) oosfile: write [1724465152, 1724466176) append 0 append lockpos 0 00000080:00200000:3.0:1787827173.142959:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421012 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.142978:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724465152, len 1024, copied 1024 00000080:00200000:3.0:1787827173.142983:0:27439:0:(rw26.c:933:ll_write_end()) page@00000000e27c5fd4[3 000000005ce0a915 2 1 00000000a23cbd2a 1 0 421012] 00000080:00200000:3.0:1787827173.142986:0:27439:0:(rw26.c:933:ll_write_end()) vmpage @00000000c4f28855 fffffc0001001 3:0 ffff9e34c3cb0cb0 421012 lru 00000080:00200000:3.0:1787827173.142991:0:27439:0:(rw26.c:933:ll_write_end()) osc-page@00000000a097fd9f 210068: 1< 0 - - > 2< 860438528 0 0 0x0 0x0 | 000000004db7037e 0000000052729811 > 3< 0 0 > 4< 0 0 8 484007936 - | - - + - > 5< - - + - | 0 - | 366 - -> 00000080:00200000:3.0:1787827173.142993:0:27439:0:(rw26.c:933:ll_write_end()) end page@00000000e27c5fd4 00000080:00200000:3.0:1787827173.142994:0:27439:0:(rw26.c:933:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1787827173.142996:0:27439:0:(vvp_io.c:1214:vvp_io_write_commit()) commit async pages: 1, from 0, to 1024 prio 0 00000008:00010000:3.0:1787827173.143005:0:27439:0:(osc_cache.c:565:osc_extent_merge()) ### extent: 000000001063fa34 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.143018:0:27439:0:(osc_cache.c:497:__osc_extent_remove()) ### extent: 000000001063fa34 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.143024:0:27439:0:(osc_cache.c:582:osc_extent_merge()) ### extent: 00000000df269729 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.143030:0:27439:0:(osc_cache.c:846:osc_extent_find()) ### extent: 00000000df269729 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.143040:0:27439:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff9e35d0e392f8, count 1, dirtied 1 00000080:00200000:3.0:1787827173.143044:0:27439:0:(vvp_io.c:1237:vvp_io_write_commit()) Committed 1 pages 1024 bytes, tot: 1024 00000080:00200000:3.0:1787827173.143046:0:27439:0:(vvp_io.c:1421:vvp_io_write_start()) oosfile: write bytes 0, result: 1024 00000020:00010000:3.0:1787827173.143060:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_release():245 00000020:00010000:3.0:1787827173.143063:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1787827173.143065:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c7d81d30 (ffff9e35da704cf8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1787827173.143069:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.143077:0:27439:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 17/0,0 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1787827173.143086:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_fini():72 00000020:00010000:3.0:1787827173.143088:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c7d81d30 (ffff9e35c614ce18/1) at cl_lock_fini():72 00000080:00200000:3.0:1787827173.143093:0:27439:0:(vvp_io.c:533:vvp_io_advance()) advancing 1024 bytes 00000080:00200000:3.0:1787827173.143096:0:27439:0:(file.c:2207:ll_file_io_generic()) Range unlock [421012, 421012] 00000080:00200000:3.0:1787827173.143100:0:27439:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787827173.143103:0:27439:0:(file.c:2240:ll_file_io_generic()) oosfile: 2 io complete with rc: 0, result: 1024, restart: 0 00000080:00200000:3.0:1787827173.143107:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724466176 i_blocks 3365288 00000080:00200000:3.0:1787827173.143110:0:27439:0:(file.c:2326:ll_file_io_generic()) iot: 2, result: 1024 00000080:00200008:3.0:1787827173.143117:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724466176, count: 1024 00000080:00200000:3.0:1787827173.143121:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421012 of 1024 to 1024 bytes 00000080:00200000:3.0:1787827173.143124:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724466176, len 1024, copied 1024 00000080:00200000:3.0:1787827173.143129:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724467200 i_blocks 3365288 00000080:00200000:3.0:1787827173.143131:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.143135:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724467200, count: 1024 00000080:00200000:3.0:1787827173.143138:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421012 of 2048 to 1024 bytes 00000080:00200000:3.0:1787827173.143140:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724467200, len 1024, copied 1024 00000080:00200000:3.0:1787827173.143143:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724468224 i_blocks 3365288 00000080:00200000:3.0:1787827173.143145:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.143149:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724468224, count: 1024 00000080:00200000:3.0:1787827173.143152:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421012 of 3072 to 1024 bytes 00000080:00200000:3.0:1787827173.143154:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724468224, len 1024, copied 1024 00000080:00200000:3.0:1787827173.143157:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724469248 i_blocks 3365288 00000080:00200000:3.0:1787827173.143159:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.143163:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724469248, count: 1024 00000080:00200000:3.0:1787827173.143166:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421013 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.143168:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 0, original count 1024 00000080:00200000:3.0:1787827173.143170:0:27439:0:(file.c:2090:ll_file_io_generic()) oosfile: write ppos: 1724469248, bytes: 1024 00000080:00200000:3.0:1787827173.143172:0:27439:0:(file.c:1848:ll_io_set_mirror()) oosfile: desiginated mirror: 0 00000020:00200000:3.0:1787827173.143176:0:27439:0:(cl_io.c:218:cl_io_rw_init()) header@000000007788600a[0x110, 368566, [0x200000402:0x27:0x0] hash] 00000020:00200000:3.0:1787827173.143177:0:27439:0:(cl_io.c:218:cl_io_rw_init()) io range: 2 [1724469248, 1724470272) 0 0 00000080:00200000:3.0:1787827173.143180:0:27439:0:(vvp_io.c:1889:vvp_io_init()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1787827173.143184:0:27439:0:(file.c:2170:ll_file_io_generic()) Range lock [421013, 421013] 00020000:00200000:3.0:1787827173.143187:0:27439:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 411 chunk: [1724469248, 1724470272) 1724470272, 1024 00020000:00200000:3.0:1787827173.143190:0:27439:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1787827173.143195:0:27439:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [421013, 421013] 00000008:00010000:3.0:1787827173.143200:0:27439:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1787827173.143203:0:27439:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff9e35c7d81d30, osc lock ffff9e35c2ddcc30, flags 0x0 00010000:00010000:3.0:1787827173.143211:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.143219:0:27439:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (860442624 860446719) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.143227:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 20/0,2 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.143234:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 20/0,2 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.143241:0:27439:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 19/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.143248:0:27439:0:(vvp_io.c:1339:vvp_io_write_start()) oosfile: write [1724469248, 1724470272) append 0 append lockpos 0 00000080:00200000:3.0:1787827173.143252:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421013 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.143269:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724469248, len 1024, copied 1024 00000080:00200000:3.0:1787827173.143274:0:27439:0:(rw26.c:933:ll_write_end()) page@00000000170c5850[3 000000005ce0a915 2 1 00000000a23cbd2a 1 0 421013] 00000080:00200000:3.0:1787827173.143276:0:27439:0:(rw26.c:933:ll_write_end()) vmpage @00000000400c9720 fffffc0001001 3:0 ffff9e34c3cb01d0 421013 lru 00000080:00200000:3.0:1787827173.143281:0:27439:0:(rw26.c:933:ll_write_end()) osc-page@000000005198d418 210069: 1< 0 - - > 2< 860442624 0 0 0x0 0x0 | 000000004db7037e 0000000052729811 > 3< 0 0 > 4< 0 0 8 484003840 - | - - + - > 5< - - + - | 0 - | 367 - -> 00000080:00200000:3.0:1787827173.143283:0:27439:0:(rw26.c:933:ll_write_end()) end page@00000000170c5850 00000080:00200000:3.0:1787827173.143284:0:27439:0:(rw26.c:933:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1787827173.143287:0:27439:0:(vvp_io.c:1214:vvp_io_write_commit()) commit async pages: 1, from 0, to 1024 prio 0 00000008:00010000:3.0:1787827173.143295:0:27439:0:(osc_cache.c:565:osc_extent_merge()) ### extent: 000000001063fa34 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.143307:0:27439:0:(osc_cache.c:497:__osc_extent_remove()) ### extent: 000000001063fa34 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.143314:0:27439:0:(osc_cache.c:582:osc_extent_merge()) ### extent: 00000000df269729 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.143321:0:27439:0:(osc_cache.c:846:osc_extent_find()) ### extent: 00000000df269729 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.143330:0:27439:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff9e35d0e392f8, count 1, dirtied 1 00000080:00200000:3.0:1787827173.143333:0:27439:0:(vvp_io.c:1237:vvp_io_write_commit()) Committed 1 pages 1024 bytes, tot: 1024 00000080:00200000:3.0:1787827173.143335:0:27439:0:(vvp_io.c:1421:vvp_io_write_start()) oosfile: write bytes 0, result: 1024 00000020:00010000:3.0:1787827173.143341:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_release():245 00000020:00010000:3.0:1787827173.143344:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1787827173.143346:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c7d81d30 (ffff9e35da704cf8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1787827173.143350:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.143358:0:27439:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 17/0,0 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1787827173.143367:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_fini():72 00000020:00010000:3.0:1787827173.143369:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c7d81d30 (ffff9e35c614ce18/1) at cl_lock_fini():72 00000080:00200000:3.0:1787827173.143373:0:27439:0:(vvp_io.c:533:vvp_io_advance()) advancing 1024 bytes 00000080:00200000:3.0:1787827173.143377:0:27439:0:(file.c:2207:ll_file_io_generic()) Range unlock [421013, 421013] 00000080:00200000:3.0:1787827173.143380:0:27439:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787827173.143383:0:27439:0:(file.c:2240:ll_file_io_generic()) oosfile: 2 io complete with rc: 0, result: 1024, restart: 0 00000080:00200000:3.0:1787827173.143387:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724470272 i_blocks 3365288 00000080:00200000:3.0:1787827173.143390:0:27439:0:(file.c:2326:ll_file_io_generic()) iot: 2, result: 1024 00000080:00200008:3.0:1787827173.143397:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724470272, count: 1024 00000080:00200000:3.0:1787827173.143401:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421013 of 1024 to 1024 bytes 00000080:00200000:3.0:1787827173.143404:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724470272, len 1024, copied 1024 00000080:00200000:3.0:1787827173.143408:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724471296 i_blocks 3365288 00000080:00200000:3.0:1787827173.143411:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.143414:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724471296, count: 1024 00000080:00200000:3.0:1787827173.143417:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421013 of 2048 to 1024 bytes 00000080:00200000:3.0:1787827173.143420:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724471296, len 1024, copied 1024 00000080:00200000:3.0:1787827173.143423:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724472320 i_blocks 3365288 00000080:00200000:3.0:1787827173.143425:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.143428:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724472320, count: 1024 00000080:00200000:3.0:1787827173.143431:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421013 of 3072 to 1024 bytes 00000080:00200000:3.0:1787827173.143433:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724472320, len 1024, copied 1024 00000080:00200000:3.0:1787827173.143437:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724473344 i_blocks 3365288 00000080:00200000:3.0:1787827173.143439:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.143442:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724473344, count: 1024 00000080:00200000:3.0:1787827173.143445:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421014 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.143447:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 0, original count 1024 00000080:00200000:3.0:1787827173.143449:0:27439:0:(file.c:2090:ll_file_io_generic()) oosfile: write ppos: 1724473344, bytes: 1024 00000080:00200000:3.0:1787827173.143451:0:27439:0:(file.c:1848:ll_io_set_mirror()) oosfile: desiginated mirror: 0 00000020:00200000:3.0:1787827173.143455:0:27439:0:(cl_io.c:218:cl_io_rw_init()) header@000000007788600a[0x110, 368567, [0x200000402:0x27:0x0] hash] 00000020:00200000:3.0:1787827173.143456:0:27439:0:(cl_io.c:218:cl_io_rw_init()) io range: 2 [1724473344, 1724474368) 0 0 00000080:00200000:3.0:1787827173.143459:0:27439:0:(vvp_io.c:1889:vvp_io_init()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1787827173.143464:0:27439:0:(file.c:2170:ll_file_io_generic()) Range lock [421014, 421014] 00020000:00200000:3.0:1787827173.143467:0:27439:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 411 chunk: [1724473344, 1724474368) 1724474368, 1024 00020000:00200000:3.0:1787827173.143469:0:27439:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1787827173.143473:0:27439:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [421014, 421014] 00000008:00010000:3.0:1787827173.143479:0:27439:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1787827173.143481:0:27439:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff9e35c7d81d30, osc lock ffff9e35c2ddcc30, flags 0x0 00010000:00010000:3.0:1787827173.143489:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.143497:0:27439:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (860446720 860450815) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.143506:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 20/0,2 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.143513:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 20/0,2 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.143519:0:27439:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 19/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.143525:0:27439:0:(vvp_io.c:1339:vvp_io_write_start()) oosfile: write [1724473344, 1724474368) append 0 append lockpos 0 00000080:00200000:3.0:1787827173.143530:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421014 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.143547:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724473344, len 1024, copied 1024 00000080:00200000:3.0:1787827173.143551:0:27439:0:(rw26.c:933:ll_write_end()) page@0000000036fff86b[3 000000005ce0a915 2 1 00000000a23cbd2a 1 0 421014] 00000080:00200000:3.0:1787827173.143554:0:27439:0:(rw26.c:933:ll_write_end()) vmpage @00000000bfefd33b fffffc0001001 3:0 ffff9e34c3cb0658 421014 lru 00000080:00200000:3.0:1787827173.143559:0:27439:0:(rw26.c:933:ll_write_end()) osc-page@0000000027e23eee 210070: 1< 0 - - > 2< 860446720 0 0 0x0 0x0 | 000000004db7037e 0000000052729811 > 3< 0 0 > 4< 0 0 8 483999744 - | - - + - > 5< - - + - | 0 - | 368 - -> 00000080:00200000:3.0:1787827173.143561:0:27439:0:(rw26.c:933:ll_write_end()) end page@0000000036fff86b 00000080:00200000:3.0:1787827173.143562:0:27439:0:(rw26.c:933:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1787827173.143564:0:27439:0:(vvp_io.c:1214:vvp_io_write_commit()) commit async pages: 1, from 0, to 1024 prio 0 00000008:00010000:3.0:1787827173.143572:0:27439:0:(osc_cache.c:565:osc_extent_merge()) ### extent: 000000001063fa34 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.143585:0:27439:0:(osc_cache.c:497:__osc_extent_remove()) ### extent: 000000001063fa34 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.143592:0:27439:0:(osc_cache.c:582:osc_extent_merge()) ### extent: 00000000df269729 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.143598:0:27439:0:(osc_cache.c:846:osc_extent_find()) ### extent: 00000000df269729 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.143608:0:27439:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff9e35d0e392f8, count 1, dirtied 1 00000080:00200000:3.0:1787827173.143610:0:27439:0:(vvp_io.c:1237:vvp_io_write_commit()) Committed 1 pages 1024 bytes, tot: 1024 00000080:00200000:3.0:1787827173.143612:0:27439:0:(vvp_io.c:1421:vvp_io_write_start()) oosfile: write bytes 0, result: 1024 00000020:00010000:3.0:1787827173.143617:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_release():245 00000020:00010000:3.0:1787827173.143620:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1787827173.143622:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c7d81d30 (ffff9e35da704cf8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1787827173.143626:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.143634:0:27439:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 17/0,0 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1787827173.143642:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_fini():72 00000020:00010000:3.0:1787827173.143644:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c7d81d30 (ffff9e35c614ce18/1) at cl_lock_fini():72 00000080:00200000:3.0:1787827173.143648:0:27439:0:(vvp_io.c:533:vvp_io_advance()) advancing 1024 bytes 00000080:00200000:3.0:1787827173.143650:0:27439:0:(file.c:2207:ll_file_io_generic()) Range unlock [421014, 421014] 00000080:00200000:3.0:1787827173.143654:0:27439:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787827173.143657:0:27439:0:(file.c:2240:ll_file_io_generic()) oosfile: 2 io complete with rc: 0, result: 1024, restart: 0 00000080:00200000:3.0:1787827173.143661:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724474368 i_blocks 3365288 00000080:00200000:3.0:1787827173.143664:0:27439:0:(file.c:2326:ll_file_io_generic()) iot: 2, result: 1024 00000080:00200008:3.0:1787827173.143670:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724474368, count: 1024 00000080:00200000:3.0:1787827173.143675:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421014 of 1024 to 1024 bytes 00000080:00200000:3.0:1787827173.143678:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724474368, len 1024, copied 1024 00000080:00200000:3.0:1787827173.143682:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724475392 i_blocks 3365288 00000080:00200000:3.0:1787827173.143684:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.143688:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724475392, count: 1024 00000080:00200000:3.0:1787827173.143691:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421014 of 2048 to 1024 bytes 00000080:00200000:3.0:1787827173.143693:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724475392, len 1024, copied 1024 00000080:00200000:3.0:1787827173.143697:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724476416 i_blocks 3365288 00000080:00200000:3.0:1787827173.143699:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.143702:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724476416, count: 1024 00000080:00200000:3.0:1787827173.143705:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421014 of 3072 to 1024 bytes 00000080:00200000:3.0:1787827173.143707:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724476416, len 1024, copied 1024 00000080:00200000:3.0:1787827173.143711:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724477440 i_blocks 3365288 00000080:00200000:3.0:1787827173.143713:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.143716:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724477440, count: 1024 00000080:00200000:3.0:1787827173.143719:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421015 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.143721:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 0, original count 1024 00000080:00200000:3.0:1787827173.143722:0:27439:0:(file.c:2090:ll_file_io_generic()) oosfile: write ppos: 1724477440, bytes: 1024 00000080:00200000:3.0:1787827173.143724:0:27439:0:(file.c:1848:ll_io_set_mirror()) oosfile: desiginated mirror: 0 00000020:00200000:3.0:1787827173.143728:0:27439:0:(cl_io.c:218:cl_io_rw_init()) header@000000007788600a[0x110, 368568, [0x200000402:0x27:0x0] hash] 00000020:00200000:3.0:1787827173.143729:0:27439:0:(cl_io.c:218:cl_io_rw_init()) io range: 2 [1724477440, 1724478464) 0 0 00000080:00200000:3.0:1787827173.143731:0:27439:0:(vvp_io.c:1889:vvp_io_init()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1787827173.143736:0:27439:0:(file.c:2170:ll_file_io_generic()) Range lock [421015, 421015] 00020000:00200000:3.0:1787827173.143739:0:27439:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 411 chunk: [1724477440, 1724478464) 1724478464, 1024 00020000:00200000:3.0:1787827173.143741:0:27439:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1787827173.143745:0:27439:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [421015, 421015] 00000008:00010000:3.0:1787827173.143763:0:27439:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1787827173.143766:0:27439:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff9e35c7d81d30, osc lock ffff9e35c2ddcc30, flags 0x0 00010000:00010000:3.0:1787827173.143774:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.143782:0:27439:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (860450816 860454911) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.143790:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 20/0,2 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.143816:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 20/0,2 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.143824:0:27439:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 19/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.143831:0:27439:0:(vvp_io.c:1339:vvp_io_write_start()) oosfile: write [1724477440, 1724478464) append 0 append lockpos 0 00000080:00200000:3.0:1787827173.143836:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421015 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.143852:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724477440, len 1024, copied 1024 00000080:00200000:3.0:1787827173.143857:0:27439:0:(rw26.c:933:ll_write_end()) page@00000000028ec87d[3 000000005ce0a915 2 1 00000000a23cbd2a 1 0 421015] 00000080:00200000:3.0:1787827173.143859:0:27439:0:(rw26.c:933:ll_write_end()) vmpage @00000000b4edad9e fffffc0001001 3:0 ffff9e34c3cb03a0 421015 lru 00000080:00200000:3.0:1787827173.143864:0:27439:0:(rw26.c:933:ll_write_end()) osc-page@00000000a732f118 210071: 1< 0 - - > 2< 860450816 0 0 0x0 0x0 | 000000004db7037e 0000000052729811 > 3< 0 0 > 4< 0 0 8 483995648 - | - - + - > 5< - - + - | 0 - | 369 - -> 00000080:00200000:3.0:1787827173.143866:0:27439:0:(rw26.c:933:ll_write_end()) end page@00000000028ec87d 00000080:00200000:3.0:1787827173.143868:0:27439:0:(rw26.c:933:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1787827173.143870:0:27439:0:(vvp_io.c:1214:vvp_io_write_commit()) commit async pages: 1, from 0, to 1024 prio 0 00000008:00010000:3.0:1787827173.143879:0:27439:0:(osc_cache.c:565:osc_extent_merge()) ### extent: 000000001063fa34 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.143890:0:27439:0:(osc_cache.c:497:__osc_extent_remove()) ### extent: 000000001063fa34 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.143896:0:27439:0:(osc_cache.c:582:osc_extent_merge()) ### extent: 00000000df269729 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.143902:0:27439:0:(osc_cache.c:846:osc_extent_find()) ### extent: 00000000df269729 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.143912:0:27439:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff9e35d0e392f8, count 1, dirtied 1 00000080:00200000:3.0:1787827173.143914:0:27439:0:(vvp_io.c:1237:vvp_io_write_commit()) Committed 1 pages 1024 bytes, tot: 1024 00000080:00200000:3.0:1787827173.143916:0:27439:0:(vvp_io.c:1421:vvp_io_write_start()) oosfile: write bytes 0, result: 1024 00000020:00010000:3.0:1787827173.143935:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_release():245 00000020:00010000:3.0:1787827173.143938:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1787827173.143940:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c7d81d30 (ffff9e35da704cf8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1787827173.143944:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.143951:0:27439:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 17/0,0 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1787827173.143959:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_fini():72 00000020:00010000:3.0:1787827173.143962:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c7d81d30 (ffff9e35c614ce18/1) at cl_lock_fini():72 00000080:00200000:3.0:1787827173.143966:0:27439:0:(vvp_io.c:533:vvp_io_advance()) advancing 1024 bytes 00000080:00200000:3.0:1787827173.143968:0:27439:0:(file.c:2207:ll_file_io_generic()) Range unlock [421015, 421015] 00000080:00200000:3.0:1787827173.143973:0:27439:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787827173.143976:0:27439:0:(file.c:2240:ll_file_io_generic()) oosfile: 2 io complete with rc: 0, result: 1024, restart: 0 00000080:00200000:3.0:1787827173.143981:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724478464 i_blocks 3365288 00000080:00200000:3.0:1787827173.143984:0:27439:0:(file.c:2326:ll_file_io_generic()) iot: 2, result: 1024 00000080:00200008:3.0:1787827173.143990:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724478464, count: 1024 00000080:00200000:3.0:1787827173.143994:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421015 of 1024 to 1024 bytes 00000080:00200000:3.0:1787827173.143997:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724478464, len 1024, copied 1024 00000080:00200000:3.0:1787827173.144003:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724479488 i_blocks 3365288 00000080:00200000:3.0:1787827173.144005:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.144010:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724479488, count: 1024 00000080:00200000:3.0:1787827173.144013:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421015 of 2048 to 1024 bytes 00000080:00200000:3.0:1787827173.144015:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724479488, len 1024, copied 1024 00000080:00200000:3.0:1787827173.144019:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724480512 i_blocks 3365288 00000080:00200000:3.0:1787827173.144021:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.144024:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724480512, count: 1024 00000080:00200000:3.0:1787827173.144027:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421015 of 3072 to 1024 bytes 00000080:00200000:3.0:1787827173.144029:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724480512, len 1024, copied 1024 00000080:00200000:3.0:1787827173.144032:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724481536 i_blocks 3365288 00000080:00200000:3.0:1787827173.144035:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.144038:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724481536, count: 1024 00000080:00200000:3.0:1787827173.144041:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421016 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.144043:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 0, original count 1024 00000080:00200000:3.0:1787827173.144045:0:27439:0:(file.c:2090:ll_file_io_generic()) oosfile: write ppos: 1724481536, bytes: 1024 00000080:00200000:3.0:1787827173.144047:0:27439:0:(file.c:1848:ll_io_set_mirror()) oosfile: desiginated mirror: 0 00000020:00200000:3.0:1787827173.144051:0:27439:0:(cl_io.c:218:cl_io_rw_init()) header@000000007788600a[0x110, 368569, [0x200000402:0x27:0x0] hash] 00000020:00200000:3.0:1787827173.144053:0:27439:0:(cl_io.c:218:cl_io_rw_init()) io range: 2 [1724481536, 1724482560) 0 0 00000080:00200000:3.0:1787827173.144055:0:27439:0:(vvp_io.c:1889:vvp_io_init()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1787827173.144061:0:27439:0:(file.c:2170:ll_file_io_generic()) Range lock [421016, 421016] 00020000:00200000:3.0:1787827173.144063:0:27439:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 411 chunk: [1724481536, 1724482560) 1724482560, 1024 00020000:00200000:3.0:1787827173.144066:0:27439:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1787827173.144071:0:27439:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [421016, 421016] 00000008:00010000:3.0:1787827173.144077:0:27439:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1787827173.144079:0:27439:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff9e35c7d81d30, osc lock ffff9e35c2ddcc30, flags 0x0 00010000:00010000:3.0:1787827173.144086:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.144094:0:27439:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (860454912 860459007) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.144102:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 20/0,2 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.144109:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 20/0,2 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.144116:0:27439:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 19/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.144124:0:27439:0:(vvp_io.c:1339:vvp_io_write_start()) oosfile: write [1724481536, 1724482560) append 0 append lockpos 0 00000080:00200000:3.0:1787827173.144129:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421016 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.144154:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724481536, len 1024, copied 1024 00000080:00200000:3.0:1787827173.144159:0:27439:0:(rw26.c:933:ll_write_end()) page@0000000039605aa9[3 000000005ce0a915 2 1 00000000a23cbd2a 1 0 421016] 00000080:00200000:3.0:1787827173.144162:0:27439:0:(rw26.c:933:ll_write_end()) vmpage @00000000c2d355d1 fffffc0001001 3:0 ffff9e34c3cb1910 421016 lru 00000080:00200000:3.0:1787827173.144167:0:27439:0:(rw26.c:933:ll_write_end()) osc-page@000000005cc09886 210072: 1< 0 - - > 2< 860454912 0 0 0x0 0x0 | 000000004db7037e 0000000052729811 > 3< 0 0 > 4< 0 0 8 483991552 - | - - + - > 5< - - + - | 0 - | 370 - -> 00000080:00200000:3.0:1787827173.144169:0:27439:0:(rw26.c:933:ll_write_end()) end page@0000000039605aa9 00000080:00200000:3.0:1787827173.144170:0:27439:0:(rw26.c:933:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1787827173.144172:0:27439:0:(vvp_io.c:1214:vvp_io_write_commit()) commit async pages: 1, from 0, to 1024 prio 0 00000008:00010000:3.0:1787827173.144181:0:27439:0:(osc_cache.c:565:osc_extent_merge()) ### extent: 000000001063fa34 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.144195:0:27439:0:(osc_cache.c:497:__osc_extent_remove()) ### extent: 000000001063fa34 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.144203:0:27439:0:(osc_cache.c:582:osc_extent_merge()) ### extent: 00000000df269729 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.144211:0:27439:0:(osc_cache.c:846:osc_extent_find()) ### extent: 00000000df269729 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.144221:0:27439:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff9e35d0e392f8, count 1, dirtied 1 00000080:00200000:3.0:1787827173.144224:0:27439:0:(vvp_io.c:1237:vvp_io_write_commit()) Committed 1 pages 1024 bytes, tot: 1024 00000080:00200000:3.0:1787827173.144226:0:27439:0:(vvp_io.c:1421:vvp_io_write_start()) oosfile: write bytes 0, result: 1024 00000020:00010000:3.0:1787827173.144231:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_release():245 00000020:00010000:3.0:1787827173.144233:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1787827173.144236:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c7d81d30 (ffff9e35da704cf8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1787827173.144240:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.144248:0:27439:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 17/0,0 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1787827173.144257:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_fini():72 00000020:00010000:3.0:1787827173.144259:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c7d81d30 (ffff9e35c614ce18/1) at cl_lock_fini():72 00000080:00200000:3.0:1787827173.144263:0:27439:0:(vvp_io.c:533:vvp_io_advance()) advancing 1024 bytes 00000080:00200000:3.0:1787827173.144266:0:27439:0:(file.c:2207:ll_file_io_generic()) Range unlock [421016, 421016] 00000080:00200000:3.0:1787827173.144269:0:27439:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787827173.144272:0:27439:0:(file.c:2240:ll_file_io_generic()) oosfile: 2 io complete with rc: 0, result: 1024, restart: 0 00000080:00200000:3.0:1787827173.144276:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724482560 i_blocks 3365288 00000080:00200000:3.0:1787827173.144278:0:27439:0:(file.c:2326:ll_file_io_generic()) iot: 2, result: 1024 00000080:00200008:3.0:1787827173.144285:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724482560, count: 1024 00000080:00200000:3.0:1787827173.144289:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421016 of 1024 to 1024 bytes 00000080:00200000:3.0:1787827173.144292:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724482560, len 1024, copied 1024 00000080:00200000:3.0:1787827173.144296:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724483584 i_blocks 3365288 00000080:00200000:3.0:1787827173.144298:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.144302:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724483584, count: 1024 00000080:00200000:3.0:1787827173.144305:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421016 of 2048 to 1024 bytes 00000080:00200000:3.0:1787827173.144307:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724483584, len 1024, copied 1024 00000080:00200000:3.0:1787827173.144310:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724484608 i_blocks 3365288 00000080:00200000:3.0:1787827173.144313:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.144316:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724484608, count: 1024 00000080:00200000:3.0:1787827173.144319:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421016 of 3072 to 1024 bytes 00000080:00200000:3.0:1787827173.144321:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724484608, len 1024, copied 1024 00000080:00200000:3.0:1787827173.144324:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724485632 i_blocks 3365288 00000080:00200000:3.0:1787827173.144326:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.144330:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724485632, count: 1024 00000080:00200000:3.0:1787827173.144333:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421017 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.144334:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 0, original count 1024 00000080:00200000:3.0:1787827173.144336:0:27439:0:(file.c:2090:ll_file_io_generic()) oosfile: write ppos: 1724485632, bytes: 1024 00000080:00200000:3.0:1787827173.144339:0:27439:0:(file.c:1848:ll_io_set_mirror()) oosfile: desiginated mirror: 0 00000020:00200000:3.0:1787827173.144342:0:27439:0:(cl_io.c:218:cl_io_rw_init()) header@000000007788600a[0x110, 368570, [0x200000402:0x27:0x0] hash] 00000020:00200000:3.0:1787827173.144344:0:27439:0:(cl_io.c:218:cl_io_rw_init()) io range: 2 [1724485632, 1724486656) 0 0 00000080:00200000:3.0:1787827173.144346:0:27439:0:(vvp_io.c:1889:vvp_io_init()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1787827173.144352:0:27439:0:(file.c:2170:ll_file_io_generic()) Range lock [421017, 421017] 00020000:00200000:3.0:1787827173.144355:0:27439:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 411 chunk: [1724485632, 1724486656) 1724486656, 1024 00020000:00200000:3.0:1787827173.144357:0:27439:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1787827173.144362:0:27439:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [421017, 421017] 00000008:00010000:3.0:1787827173.144367:0:27439:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1787827173.144375:0:27439:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff9e35c7d81d30, osc lock ffff9e35c2ddcc30, flags 0x0 00010000:00010000:3.0:1787827173.144383:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.144391:0:27439:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (860459008 860463103) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.144400:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 20/0,2 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.144407:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 20/0,2 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.144414:0:27439:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 19/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.144421:0:27439:0:(vvp_io.c:1339:vvp_io_write_start()) oosfile: write [1724485632, 1724486656) append 0 append lockpos 0 00000080:00200000:3.0:1787827173.144426:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421017 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.144443:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724485632, len 1024, copied 1024 00000080:00200000:3.0:1787827173.144448:0:27439:0:(rw26.c:933:ll_write_end()) page@00000000a39078f3[3 000000005ce0a915 2 1 00000000a23cbd2a 1 0 421017] 00000080:00200000:3.0:1787827173.144451:0:27439:0:(rw26.c:933:ll_write_end()) vmpage @00000000af884c6f fffffc0001001 3:0 ffff9e34c3cb10e8 421017 lru 00000080:00200000:3.0:1787827173.144457:0:27439:0:(rw26.c:933:ll_write_end()) osc-page@000000008b9991af 210073: 1< 0 - - > 2< 860459008 0 0 0x0 0x0 | 000000004db7037e 0000000052729811 > 3< 0 0 > 4< 0 0 8 483987456 - | - - + - > 5< - - + - | 0 - | 371 - -> 00000080:00200000:3.0:1787827173.144459:0:27439:0:(rw26.c:933:ll_write_end()) end page@00000000a39078f3 00000080:00200000:3.0:1787827173.144460:0:27439:0:(rw26.c:933:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1787827173.144462:0:27439:0:(vvp_io.c:1214:vvp_io_write_commit()) commit async pages: 1, from 0, to 1024 prio 0 00000008:00010000:3.0:1787827173.144471:0:27439:0:(osc_cache.c:565:osc_extent_merge()) ### extent: 000000001063fa34 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.144484:0:27439:0:(osc_cache.c:497:__osc_extent_remove()) ### extent: 000000001063fa34 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.144491:0:27439:0:(osc_cache.c:582:osc_extent_merge()) ### extent: 00000000df269729 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.144500:0:27439:0:(osc_cache.c:846:osc_extent_find()) ### extent: 00000000df269729 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.144511:0:27439:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff9e35d0e392f8, count 1, dirtied 1 00000080:00200000:3.0:1787827173.144514:0:27439:0:(vvp_io.c:1237:vvp_io_write_commit()) Committed 1 pages 1024 bytes, tot: 1024 00000080:00200000:3.0:1787827173.144517:0:27439:0:(vvp_io.c:1421:vvp_io_write_start()) oosfile: write bytes 0, result: 1024 00000020:00010000:3.0:1787827173.144523:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_release():245 00000020:00010000:3.0:1787827173.144525:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1787827173.144527:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c7d81d30 (ffff9e35da704cf8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1787827173.144532:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.144539:0:27439:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 17/0,0 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1787827173.144548:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_fini():72 00000020:00010000:3.0:1787827173.144550:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c7d81d30 (ffff9e35c614ce18/1) at cl_lock_fini():72 00000080:00200000:3.0:1787827173.144554:0:27439:0:(vvp_io.c:533:vvp_io_advance()) advancing 1024 bytes 00000080:00200000:3.0:1787827173.144557:0:27439:0:(file.c:2207:ll_file_io_generic()) Range unlock [421017, 421017] 00000080:00200000:3.0:1787827173.144561:0:27439:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787827173.144564:0:27439:0:(file.c:2240:ll_file_io_generic()) oosfile: 2 io complete with rc: 0, result: 1024, restart: 0 00000080:00200000:3.0:1787827173.144568:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724486656 i_blocks 3365288 00000080:00200000:3.0:1787827173.144571:0:27439:0:(file.c:2326:ll_file_io_generic()) iot: 2, result: 1024 00000080:00200008:3.0:1787827173.144578:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724486656, count: 1024 00000080:00200000:3.0:1787827173.144582:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421017 of 1024 to 1024 bytes 00000080:00200000:3.0:1787827173.144585:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724486656, len 1024, copied 1024 00000080:00200000:3.0:1787827173.144590:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724487680 i_blocks 3365288 00000080:00200000:3.0:1787827173.144592:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.144595:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724487680, count: 1024 00000080:00200000:3.0:1787827173.144598:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421017 of 2048 to 1024 bytes 00000080:00200000:3.0:1787827173.144600:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724487680, len 1024, copied 1024 00000080:00200000:3.0:1787827173.144604:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724488704 i_blocks 3365288 00000080:00200000:3.0:1787827173.144606:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.144609:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724488704, count: 1024 00000080:00200000:3.0:1787827173.144612:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421017 of 3072 to 1024 bytes 00000080:00200000:3.0:1787827173.144614:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724488704, len 1024, copied 1024 00000080:00200000:3.0:1787827173.144617:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724489728 i_blocks 3365288 00000080:00200000:3.0:1787827173.144619:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.144622:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724489728, count: 1024 00000080:00200000:3.0:1787827173.144625:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421018 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.144627:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 0, original count 1024 00000080:00200000:3.0:1787827173.144629:0:27439:0:(file.c:2090:ll_file_io_generic()) oosfile: write ppos: 1724489728, bytes: 1024 00000080:00200000:3.0:1787827173.144631:0:27439:0:(file.c:1848:ll_io_set_mirror()) oosfile: desiginated mirror: 0 00000020:00200000:3.0:1787827173.144634:0:27439:0:(cl_io.c:218:cl_io_rw_init()) header@000000007788600a[0x110, 368571, [0x200000402:0x27:0x0] hash] 00000020:00200000:3.0:1787827173.144636:0:27439:0:(cl_io.c:218:cl_io_rw_init()) io range: 2 [1724489728, 1724490752) 0 0 00000080:00200000:3.0:1787827173.144638:0:27439:0:(vvp_io.c:1889:vvp_io_init()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1787827173.144643:0:27439:0:(file.c:2170:ll_file_io_generic()) Range lock [421018, 421018] 00020000:00200000:3.0:1787827173.144645:0:27439:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 411 chunk: [1724489728, 1724490752) 1724490752, 1024 00020000:00200000:3.0:1787827173.144648:0:27439:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1787827173.144652:0:27439:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [421018, 421018] 00000008:00010000:3.0:1787827173.144657:0:27439:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1787827173.144660:0:27439:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff9e35c7d81d30, osc lock ffff9e35c2ddcc30, flags 0x0 00010000:00010000:3.0:1787827173.144667:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.144676:0:27439:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (860463104 860467199) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.144685:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 20/0,2 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.144693:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 20/0,2 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.144700:0:27439:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 19/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.144707:0:27439:0:(vvp_io.c:1339:vvp_io_write_start()) oosfile: write [1724489728, 1724490752) append 0 append lockpos 0 00000080:00200000:3.0:1787827173.144711:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421018 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.144726:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724489728, len 1024, copied 1024 00000080:00200000:3.0:1787827173.144730:0:27439:0:(rw26.c:933:ll_write_end()) page@0000000000879cb8[3 000000005ce0a915 2 1 00000000a23cbd2a 1 0 421018] 00000080:00200000:3.0:1787827173.144733:0:27439:0:(rw26.c:933:ll_write_end()) vmpage @000000006d3958fb fffffc0001001 3:0 ffff9e34c3cb1cb0 421018 lru 00000080:00200000:3.0:1787827173.144739:0:27439:0:(rw26.c:933:ll_write_end()) osc-page@00000000761f9e95 210074: 1< 0 - - > 2< 860463104 0 0 0x0 0x0 | 000000004db7037e 0000000052729811 > 3< 0 0 > 4< 0 0 8 483983360 - | - - + - > 5< - - + - | 0 - | 372 - -> 00000080:00200000:3.0:1787827173.144741:0:27439:0:(rw26.c:933:ll_write_end()) end page@0000000000879cb8 00000080:00200000:3.0:1787827173.144742:0:27439:0:(rw26.c:933:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1787827173.144744:0:27439:0:(vvp_io.c:1214:vvp_io_write_commit()) commit async pages: 1, from 0, to 1024 prio 0 00000008:00010000:3.0:1787827173.144751:0:27439:0:(osc_cache.c:565:osc_extent_merge()) ### extent: 000000001063fa34 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.144760:0:27439:0:(osc_cache.c:497:__osc_extent_remove()) ### extent: 000000001063fa34 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.144764:0:27439:0:(osc_cache.c:582:osc_extent_merge()) ### extent: 00000000df269729 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.144768:0:27439:0:(osc_cache.c:846:osc_extent_find()) ### extent: 00000000df269729 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.144775:0:27439:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff9e35d0e392f8, count 1, dirtied 1 00000080:00200000:3.0:1787827173.144776:0:27439:0:(vvp_io.c:1237:vvp_io_write_commit()) Committed 1 pages 1024 bytes, tot: 1024 00000080:00200000:3.0:1787827173.144778:0:27439:0:(vvp_io.c:1421:vvp_io_write_start()) oosfile: write bytes 0, result: 1024 00000020:00010000:3.0:1787827173.144781:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_release():245 00000020:00010000:3.0:1787827173.144783:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1787827173.144784:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c7d81d30 (ffff9e35da704cf8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1787827173.144787:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.144791:0:27439:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 17/0,0 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1787827173.144813:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_fini():72 00000020:00010000:3.0:1787827173.144815:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c7d81d30 (ffff9e35c614ce18/1) at cl_lock_fini():72 00000080:00200000:3.0:1787827173.144818:0:27439:0:(vvp_io.c:533:vvp_io_advance()) advancing 1024 bytes 00000080:00200000:3.0:1787827173.144820:0:27439:0:(file.c:2207:ll_file_io_generic()) Range unlock [421018, 421018] 00000080:00200000:3.0:1787827173.144823:0:27439:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787827173.144825:0:27439:0:(file.c:2240:ll_file_io_generic()) oosfile: 2 io complete with rc: 0, result: 1024, restart: 0 00000080:00200000:3.0:1787827173.144828:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724490752 i_blocks 3365288 00000080:00200000:3.0:1787827173.144830:0:27439:0:(file.c:2326:ll_file_io_generic()) iot: 2, result: 1024 00000080:00200008:3.0:1787827173.144835:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724490752, count: 1024 00000080:00200000:3.0:1787827173.144839:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421018 of 1024 to 1024 bytes 00000080:00200000:3.0:1787827173.144841:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724490752, len 1024, copied 1024 00000080:00200000:3.0:1787827173.144844:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724491776 i_blocks 3365288 00000080:00200000:3.0:1787827173.144845:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.144847:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724491776, count: 1024 00000080:00200000:3.0:1787827173.144849:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421018 of 2048 to 1024 bytes 00000080:00200000:3.0:1787827173.144851:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724491776, len 1024, copied 1024 00000080:00200000:3.0:1787827173.144853:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724492800 i_blocks 3365288 00000080:00200000:3.0:1787827173.144854:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.144856:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724492800, count: 1024 00000080:00200000:3.0:1787827173.144858:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421018 of 3072 to 1024 bytes 00000080:00200000:3.0:1787827173.144859:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724492800, len 1024, copied 1024 00000080:00200000:3.0:1787827173.144861:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724493824 i_blocks 3365288 00000080:00200000:3.0:1787827173.144862:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.144864:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724493824, count: 1024 00000080:00200000:3.0:1787827173.144866:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421019 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.144867:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 0, original count 1024 00000080:00200000:3.0:1787827173.144868:0:27439:0:(file.c:2090:ll_file_io_generic()) oosfile: write ppos: 1724493824, bytes: 1024 00000080:00200000:3.0:1787827173.144870:0:27439:0:(file.c:1848:ll_io_set_mirror()) oosfile: desiginated mirror: 0 00000020:00200000:3.0:1787827173.144872:0:27439:0:(cl_io.c:218:cl_io_rw_init()) header@000000007788600a[0x110, 368572, [0x200000402:0x27:0x0] hash] 00000020:00200000:3.0:1787827173.144873:0:27439:0:(cl_io.c:218:cl_io_rw_init()) io range: 2 [1724493824, 1724494848) 0 0 00000080:00200000:3.0:1787827173.144874:0:27439:0:(vvp_io.c:1889:vvp_io_init()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1787827173.144878:0:27439:0:(file.c:2170:ll_file_io_generic()) Range lock [421019, 421019] 00020000:00200000:3.0:1787827173.144880:0:27439:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 411 chunk: [1724493824, 1724494848) 1724494848, 1024 00020000:00200000:3.0:1787827173.144881:0:27439:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1787827173.144884:0:27439:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [421019, 421019] 00000008:00010000:3.0:1787827173.144889:0:27439:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1787827173.144890:0:27439:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff9e35c7d81d30, osc lock ffff9e35c2ddcc30, flags 0x0 00010000:00010000:3.0:1787827173.144896:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.144904:0:27439:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (860467200 860471295) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.144913:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 20/0,2 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.144931:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 20/0,2 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.144939:0:27439:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 19/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.144947:0:27439:0:(vvp_io.c:1339:vvp_io_write_start()) oosfile: write [1724493824, 1724494848) append 0 append lockpos 0 00000080:00200000:3.0:1787827173.144953:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421019 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.144970:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724493824, len 1024, copied 1024 00000080:00200000:3.0:1787827173.144973:0:27439:0:(rw26.c:933:ll_write_end()) page@0000000047f96128[3 000000005ce0a915 2 1 00000000a23cbd2a 1 0 421019] 00000080:00200000:3.0:1787827173.144975:0:27439:0:(rw26.c:933:ll_write_end()) vmpage @000000003608694c fffffc0001001 3:0 ffff9e34c3cb11d0 421019 lru 00000080:00200000:3.0:1787827173.144979:0:27439:0:(rw26.c:933:ll_write_end()) osc-page@00000000fc6d2110 210075: 1< 0 - - > 2< 860467200 0 0 0x0 0x0 | 000000004db7037e 0000000052729811 > 3< 0 0 > 4< 0 0 8 483979264 - | - - + - > 5< - - + - | 0 - | 373 - -> 00000080:00200000:3.0:1787827173.144980:0:27439:0:(rw26.c:933:ll_write_end()) end page@0000000047f96128 00000080:00200000:3.0:1787827173.144981:0:27439:0:(rw26.c:933:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1787827173.144982:0:27439:0:(vvp_io.c:1214:vvp_io_write_commit()) commit async pages: 1, from 0, to 1024 prio 0 00000008:00010000:3.0:1787827173.144988:0:27439:0:(osc_cache.c:565:osc_extent_merge()) ### extent: 000000001063fa34 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.144997:0:27439:0:(osc_cache.c:497:__osc_extent_remove()) ### extent: 000000001063fa34 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.145002:0:27439:0:(osc_cache.c:582:osc_extent_merge()) ### extent: 00000000df269729 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.145006:0:27439:0:(osc_cache.c:846:osc_extent_find()) ### extent: 00000000df269729 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.145013:0:27439:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff9e35d0e392f8, count 1, dirtied 1 00000080:00200000:3.0:1787827173.145014:0:27439:0:(vvp_io.c:1237:vvp_io_write_commit()) Committed 1 pages 1024 bytes, tot: 1024 00000080:00200000:3.0:1787827173.145016:0:27439:0:(vvp_io.c:1421:vvp_io_write_start()) oosfile: write bytes 0, result: 1024 00000020:00010000:3.0:1787827173.145019:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_release():245 00000020:00010000:3.0:1787827173.145021:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1787827173.145022:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c7d81d30 (ffff9e35da704cf8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1787827173.145025:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.145029:0:27439:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 17/0,0 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1787827173.145035:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_fini():72 00000020:00010000:3.0:1787827173.145037:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c7d81d30 (ffff9e35c614ce18/1) at cl_lock_fini():72 00000080:00200000:3.0:1787827173.145039:0:27439:0:(vvp_io.c:533:vvp_io_advance()) advancing 1024 bytes 00000080:00200000:3.0:1787827173.145041:0:27439:0:(file.c:2207:ll_file_io_generic()) Range unlock [421019, 421019] 00000080:00200000:3.0:1787827173.145044:0:27439:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787827173.145046:0:27439:0:(file.c:2240:ll_file_io_generic()) oosfile: 2 io complete with rc: 0, result: 1024, restart: 0 00000080:00200000:3.0:1787827173.145049:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724494848 i_blocks 3365288 00000080:00200000:3.0:1787827173.145050:0:27439:0:(file.c:2326:ll_file_io_generic()) iot: 2, result: 1024 00000080:00200008:3.0:1787827173.145056:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724494848, count: 1024 00000080:00200000:3.0:1787827173.145059:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421019 of 1024 to 1024 bytes 00000080:00200000:3.0:1787827173.145061:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724494848, len 1024, copied 1024 00000080:00200000:3.0:1787827173.145064:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724495872 i_blocks 3365288 00000080:00200000:3.0:1787827173.145066:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.145068:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724495872, count: 1024 00000080:00200000:3.0:1787827173.145070:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421019 of 2048 to 1024 bytes 00000080:00200000:3.0:1787827173.145071:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724495872, len 1024, copied 1024 00000080:00200000:3.0:1787827173.145073:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724496896 i_blocks 3365288 00000080:00200000:3.0:1787827173.145075:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.145077:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724496896, count: 1024 00000080:00200000:3.0:1787827173.145080:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421019 of 3072 to 1024 bytes 00000080:00200000:3.0:1787827173.145082:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724496896, len 1024, copied 1024 00000080:00200000:3.0:1787827173.145085:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724497920 i_blocks 3365288 00000080:00200000:3.0:1787827173.145087:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.145091:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724497920, count: 1024 00000080:00200000:3.0:1787827173.145094:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421020 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.145096:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 0, original count 1024 00000080:00200000:3.0:1787827173.145098:0:27439:0:(file.c:2090:ll_file_io_generic()) oosfile: write ppos: 1724497920, bytes: 1024 00000080:00200000:3.0:1787827173.145101:0:27439:0:(file.c:1848:ll_io_set_mirror()) oosfile: desiginated mirror: 0 00000020:00200000:3.0:1787827173.145104:0:27439:0:(cl_io.c:218:cl_io_rw_init()) header@000000007788600a[0x110, 368573, [0x200000402:0x27:0x0] hash] 00000020:00200000:3.0:1787827173.145106:0:27439:0:(cl_io.c:218:cl_io_rw_init()) io range: 2 [1724497920, 1724498944) 0 0 00000080:00200000:3.0:1787827173.145108:0:27439:0:(vvp_io.c:1889:vvp_io_init()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1787827173.145113:0:27439:0:(file.c:2170:ll_file_io_generic()) Range lock [421020, 421020] 00020000:00200000:3.0:1787827173.145116:0:27439:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 411 chunk: [1724497920, 1724498944) 1724498944, 1024 00020000:00200000:3.0:1787827173.145119:0:27439:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1787827173.145124:0:27439:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [421020, 421020] 00000008:00010000:3.0:1787827173.145129:0:27439:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1787827173.145131:0:27439:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff9e35c7d81d30, osc lock ffff9e35c2ddcc30, flags 0x0 00010000:00010000:3.0:1787827173.145139:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.145148:0:27439:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (860471296 860475391) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.145157:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 20/0,2 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.145165:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 20/0,2 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.145173:0:27439:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 19/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.145179:0:27439:0:(vvp_io.c:1339:vvp_io_write_start()) oosfile: write [1724497920, 1724498944) append 0 append lockpos 0 00000080:00200000:3.0:1787827173.145183:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421020 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.145199:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724497920, len 1024, copied 1024 00000080:00200000:3.0:1787827173.145203:0:27439:0:(rw26.c:933:ll_write_end()) page@00000000c7ccdea7[3 000000005ce0a915 2 1 00000000a23cbd2a 1 0 421020] 00000080:00200000:3.0:1787827173.145206:0:27439:0:(rw26.c:933:ll_write_end()) vmpage @00000000aaddf36e fffffc0001001 3:0 ffff9e34c3cb1658 421020 lru 00000080:00200000:3.0:1787827173.145212:0:27439:0:(rw26.c:933:ll_write_end()) osc-page@000000006a9f7e3c 210076: 1< 0 - - > 2< 860471296 0 0 0x0 0x0 | 000000004db7037e 0000000052729811 > 3< 0 0 > 4< 0 0 8 483975168 - | - - + - > 5< - - + - | 0 - | 374 - -> 00000080:00200000:3.0:1787827173.145214:0:27439:0:(rw26.c:933:ll_write_end()) end page@00000000c7ccdea7 00000080:00200000:3.0:1787827173.145215:0:27439:0:(rw26.c:933:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1787827173.145217:0:27439:0:(vvp_io.c:1214:vvp_io_write_commit()) commit async pages: 1, from 0, to 1024 prio 0 00000008:00010000:3.0:1787827173.145225:0:27439:0:(osc_cache.c:565:osc_extent_merge()) ### extent: 000000001063fa34 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.145238:0:27439:0:(osc_cache.c:497:__osc_extent_remove()) ### extent: 000000001063fa34 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.145246:0:27439:0:(osc_cache.c:582:osc_extent_merge()) ### extent: 00000000df269729 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.145253:0:27439:0:(osc_cache.c:846:osc_extent_find()) ### extent: 00000000df269729 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.145263:0:27439:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff9e35d0e392f8, count 1, dirtied 1 00000080:00200000:3.0:1787827173.145265:0:27439:0:(vvp_io.c:1237:vvp_io_write_commit()) Committed 1 pages 1024 bytes, tot: 1024 00000080:00200000:3.0:1787827173.145267:0:27439:0:(vvp_io.c:1421:vvp_io_write_start()) oosfile: write bytes 0, result: 1024 00000020:00010000:3.0:1787827173.145272:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_release():245 00000020:00010000:3.0:1787827173.145274:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1787827173.145277:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c7d81d30 (ffff9e35da704cf8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1787827173.145281:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.145289:0:27439:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 17/0,0 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1787827173.145297:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_fini():72 00000020:00010000:3.0:1787827173.145300:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c7d81d30 (ffff9e35c614ce18/1) at cl_lock_fini():72 00000080:00200000:3.0:1787827173.145304:0:27439:0:(vvp_io.c:533:vvp_io_advance()) advancing 1024 bytes 00000080:00200000:3.0:1787827173.145306:0:27439:0:(file.c:2207:ll_file_io_generic()) Range unlock [421020, 421020] 00000080:00200000:3.0:1787827173.145310:0:27439:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787827173.145313:0:27439:0:(file.c:2240:ll_file_io_generic()) oosfile: 2 io complete with rc: 0, result: 1024, restart: 0 00000080:00200000:3.0:1787827173.145317:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724498944 i_blocks 3365288 00000080:00200000:3.0:1787827173.145319:0:27439:0:(file.c:2326:ll_file_io_generic()) iot: 2, result: 1024 00000080:00200008:3.0:1787827173.145326:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724498944, count: 1024 00000080:00200000:3.0:1787827173.145331:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421020 of 1024 to 1024 bytes 00000080:00200000:3.0:1787827173.145334:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724498944, len 1024, copied 1024 00000080:00200000:3.0:1787827173.145339:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724499968 i_blocks 3365288 00000080:00200000:3.0:1787827173.145342:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.145346:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724499968, count: 1024 00000080:00200000:3.0:1787827173.145349:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421020 of 2048 to 1024 bytes 00000080:00200000:3.0:1787827173.145352:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724499968, len 1024, copied 1024 00000080:00200000:3.0:1787827173.145355:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724500992 i_blocks 3365288 00000080:00200000:3.0:1787827173.145358:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.145362:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724500992, count: 1024 00000080:00200000:3.0:1787827173.145365:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421020 of 3072 to 1024 bytes 00000080:00200000:3.0:1787827173.145367:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724500992, len 1024, copied 1024 00000080:00200000:3.0:1787827173.145370:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724502016 i_blocks 3365288 00000080:00200000:3.0:1787827173.145373:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.145376:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724502016, count: 1024 00000080:00200000:3.0:1787827173.145379:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421021 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.145382:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 0, original count 1024 00000080:00200000:3.0:1787827173.145384:0:27439:0:(file.c:2090:ll_file_io_generic()) oosfile: write ppos: 1724502016, bytes: 1024 00000080:00200000:3.0:1787827173.145386:0:27439:0:(file.c:1848:ll_io_set_mirror()) oosfile: desiginated mirror: 0 00000020:00200000:3.0:1787827173.145390:0:27439:0:(cl_io.c:218:cl_io_rw_init()) header@000000007788600a[0x110, 368574, [0x200000402:0x27:0x0] hash] 00000020:00200000:3.0:1787827173.145391:0:27439:0:(cl_io.c:218:cl_io_rw_init()) io range: 2 [1724502016, 1724503040) 0 0 00000080:00200000:3.0:1787827173.145394:0:27439:0:(vvp_io.c:1889:vvp_io_init()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1787827173.145399:0:27439:0:(file.c:2170:ll_file_io_generic()) Range lock [421021, 421021] 00020000:00200000:3.0:1787827173.145402:0:27439:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 411 chunk: [1724502016, 1724503040) 1724503040, 1024 00020000:00200000:3.0:1787827173.145404:0:27439:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1787827173.145409:0:27439:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [421021, 421021] 00000008:00010000:3.0:1787827173.145416:0:27439:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1787827173.145418:0:27439:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff9e35c7d81d30, osc lock ffff9e35c2ddcc30, flags 0x0 00010000:00010000:3.0:1787827173.145426:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.145436:0:27439:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (860475392 860479487) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.145444:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 20/0,2 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.145451:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 20/0,2 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.145458:0:27439:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 19/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.145466:0:27439:0:(vvp_io.c:1339:vvp_io_write_start()) oosfile: write [1724502016, 1724503040) append 0 append lockpos 0 00000080:00200000:3.0:1787827173.145471:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421021 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.145489:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724502016, len 1024, copied 1024 00000080:00200000:3.0:1787827173.145495:0:27439:0:(rw26.c:933:ll_write_end()) page@000000005a9b25a8[3 000000005ce0a915 2 1 00000000a23cbd2a 1 0 421021] 00000080:00200000:3.0:1787827173.145498:0:27439:0:(rw26.c:933:ll_write_end()) vmpage @0000000052c8c48c fffffc0001001 3:0 ffff9e34c3cb13a0 421021 lru 00000080:00200000:3.0:1787827173.145503:0:27439:0:(rw26.c:933:ll_write_end()) osc-page@00000000cfa8a6fc 210077: 1< 0 - - > 2< 860475392 0 0 0x0 0x0 | 000000004db7037e 0000000052729811 > 3< 0 0 > 4< 0 0 8 483971072 - | - - + - > 5< - - + - | 0 - | 375 - -> 00000080:00200000:3.0:1787827173.145505:0:27439:0:(rw26.c:933:ll_write_end()) end page@000000005a9b25a8 00000080:00200000:3.0:1787827173.145506:0:27439:0:(rw26.c:933:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1787827173.145508:0:27439:0:(vvp_io.c:1214:vvp_io_write_commit()) commit async pages: 1, from 0, to 1024 prio 0 00000008:00010000:3.0:1787827173.145517:0:27439:0:(osc_cache.c:565:osc_extent_merge()) ### extent: 000000001063fa34 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.145531:0:27439:0:(osc_cache.c:497:__osc_extent_remove()) ### extent: 000000001063fa34 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.145537:0:27439:0:(osc_cache.c:582:osc_extent_merge()) ### extent: 00000000df269729 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.145545:0:27439:0:(osc_cache.c:846:osc_extent_find()) ### extent: 00000000df269729 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.145556:0:27439:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff9e35d0e392f8, count 1, dirtied 1 00000080:00200000:3.0:1787827173.145558:0:27439:0:(vvp_io.c:1237:vvp_io_write_commit()) Committed 1 pages 1024 bytes, tot: 1024 00000080:00200000:3.0:1787827173.145561:0:27439:0:(vvp_io.c:1421:vvp_io_write_start()) oosfile: write bytes 0, result: 1024 00000020:00010000:3.0:1787827173.145566:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_release():245 00000020:00010000:3.0:1787827173.145569:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1787827173.145572:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c7d81d30 (ffff9e35da704cf8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1787827173.145576:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.145585:0:27439:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 17/0,0 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1787827173.145594:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_fini():72 00000020:00010000:3.0:1787827173.145597:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c7d81d30 (ffff9e35c614ce18/1) at cl_lock_fini():72 00000080:00200000:3.0:1787827173.145603:0:27439:0:(vvp_io.c:533:vvp_io_advance()) advancing 1024 bytes 00000080:00200000:3.0:1787827173.145606:0:27439:0:(file.c:2207:ll_file_io_generic()) Range unlock [421021, 421021] 00000080:00200000:3.0:1787827173.145610:0:27439:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787827173.145613:0:27439:0:(file.c:2240:ll_file_io_generic()) oosfile: 2 io complete with rc: 0, result: 1024, restart: 0 00000080:00200000:3.0:1787827173.145618:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724503040 i_blocks 3365288 00000080:00200000:3.0:1787827173.145621:0:27439:0:(file.c:2326:ll_file_io_generic()) iot: 2, result: 1024 00000080:00200008:3.0:1787827173.145628:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724503040, count: 1024 00000080:00200000:3.0:1787827173.145633:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421021 of 1024 to 1024 bytes 00000080:00200000:3.0:1787827173.145636:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724503040, len 1024, copied 1024 00000080:00200000:3.0:1787827173.145641:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724504064 i_blocks 3365288 00000080:00200000:3.0:1787827173.145643:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.145647:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724504064, count: 1024 00000080:00200000:3.0:1787827173.145650:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421021 of 2048 to 1024 bytes 00000080:00200000:3.0:1787827173.145652:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724504064, len 1024, copied 1024 00000080:00200000:3.0:1787827173.145655:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724505088 i_blocks 3365288 00000080:00200000:3.0:1787827173.145657:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.145661:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724505088, count: 1024 00000080:00200000:3.0:1787827173.145663:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421021 of 3072 to 1024 bytes 00000080:00200000:3.0:1787827173.145666:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724505088, len 1024, copied 1024 00000080:00200000:3.0:1787827173.145669:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724506112 i_blocks 3365288 00000080:00200000:3.0:1787827173.145672:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.145675:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724506112, count: 1024 00000080:00200000:3.0:1787827173.145678:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421022 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.145680:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 0, original count 1024 00000080:00200000:3.0:1787827173.145682:0:27439:0:(file.c:2090:ll_file_io_generic()) oosfile: write ppos: 1724506112, bytes: 1024 00000080:00200000:3.0:1787827173.145684:0:27439:0:(file.c:1848:ll_io_set_mirror()) oosfile: desiginated mirror: 0 00000020:00200000:3.0:1787827173.145688:0:27439:0:(cl_io.c:218:cl_io_rw_init()) header@000000007788600a[0x110, 368575, [0x200000402:0x27:0x0] hash] 00000020:00200000:3.0:1787827173.145689:0:27439:0:(cl_io.c:218:cl_io_rw_init()) io range: 2 [1724506112, 1724507136) 0 0 00000080:00200000:3.0:1787827173.145691:0:27439:0:(vvp_io.c:1889:vvp_io_init()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1787827173.145697:0:27439:0:(file.c:2170:ll_file_io_generic()) Range lock [421022, 421022] 00020000:00200000:3.0:1787827173.145699:0:27439:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 411 chunk: [1724506112, 1724507136) 1724507136, 1024 00020000:00200000:3.0:1787827173.145702:0:27439:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1787827173.145706:0:27439:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [421022, 421022] 00000008:00010000:3.0:1787827173.145712:0:27439:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1787827173.145715:0:27439:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff9e35c7d81d30, osc lock ffff9e35c2ddcc30, flags 0x0 00010000:00010000:3.0:1787827173.145723:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.145732:0:27439:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (860479488 860483583) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.145742:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 20/0,2 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.145750:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 20/0,2 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.145758:0:27439:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 19/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.145765:0:27439:0:(vvp_io.c:1339:vvp_io_write_start()) oosfile: write [1724506112, 1724507136) append 0 append lockpos 0 00000080:00200000:3.0:1787827173.145770:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421022 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.145787:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724506112, len 1024, copied 1024 00000080:00200000:3.0:1787827173.145792:0:27439:0:(rw26.c:933:ll_write_end()) page@0000000022030f1a[3 000000005ce0a915 2 1 00000000a23cbd2a 1 0 421022] 00000080:00200000:3.0:1787827173.145795:0:27439:0:(rw26.c:933:ll_write_end()) vmpage @0000000059b66a39 fffffc0001001 3:0 ffff9e34c3cb1d98 421022 lru 00000080:00200000:3.0:1787827173.145824:0:27439:0:(rw26.c:933:ll_write_end()) osc-page@000000007f317921 210078: 1< 0 - - > 2< 860479488 0 0 0x0 0x0 | 000000004db7037e 0000000052729811 > 3< 0 0 > 4< 0 0 8 483966976 - | - - + - > 5< - - + - | 0 - | 376 - -> 00000080:00200000:3.0:1787827173.145826:0:27439:0:(rw26.c:933:ll_write_end()) end page@0000000022030f1a 00000080:00200000:3.0:1787827173.145827:0:27439:0:(rw26.c:933:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1787827173.145830:0:27439:0:(vvp_io.c:1214:vvp_io_write_commit()) commit async pages: 1, from 0, to 1024 prio 0 00000008:00010000:3.0:1787827173.145839:0:27439:0:(osc_cache.c:565:osc_extent_merge()) ### extent: 000000001063fa34 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.145853:0:27439:0:(osc_cache.c:497:__osc_extent_remove()) ### extent: 000000001063fa34 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.145861:0:27439:0:(osc_cache.c:582:osc_extent_merge()) ### extent: 00000000df269729 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.145869:0:27439:0:(osc_cache.c:846:osc_extent_find()) ### extent: 00000000df269729 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.145881:0:27439:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff9e35d0e392f8, count 1, dirtied 1 00000080:00200000:3.0:1787827173.145884:0:27439:0:(vvp_io.c:1237:vvp_io_write_commit()) Committed 1 pages 1024 bytes, tot: 1024 00000080:00200000:3.0:1787827173.145886:0:27439:0:(vvp_io.c:1421:vvp_io_write_start()) oosfile: write bytes 0, result: 1024 00000020:00010000:3.0:1787827173.145892:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_release():245 00000020:00010000:3.0:1787827173.145895:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1787827173.145897:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c7d81d30 (ffff9e35da704cf8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1787827173.145901:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.145910:0:27439:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 17/0,0 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1787827173.145931:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_fini():72 00000020:00010000:3.0:1787827173.145934:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c7d81d30 (ffff9e35c614ce18/1) at cl_lock_fini():72 00000080:00200000:3.0:1787827173.145939:0:27439:0:(vvp_io.c:533:vvp_io_advance()) advancing 1024 bytes 00000080:00200000:3.0:1787827173.145941:0:27439:0:(file.c:2207:ll_file_io_generic()) Range unlock [421022, 421022] 00000080:00200000:3.0:1787827173.145945:0:27439:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787827173.145948:0:27439:0:(file.c:2240:ll_file_io_generic()) oosfile: 2 io complete with rc: 0, result: 1024, restart: 0 00000080:00200000:3.0:1787827173.145952:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724507136 i_blocks 3365288 00000080:00200000:3.0:1787827173.145954:0:27439:0:(file.c:2326:ll_file_io_generic()) iot: 2, result: 1024 00000080:00200008:3.0:1787827173.145962:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724507136, count: 1024 00000080:00200000:3.0:1787827173.145966:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421022 of 1024 to 1024 bytes 00000080:00200000:3.0:1787827173.145970:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724507136, len 1024, copied 1024 00000080:00200000:3.0:1787827173.145975:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724508160 i_blocks 3365288 00000080:00200000:3.0:1787827173.145977:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.145981:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724508160, count: 1024 00000080:00200000:3.0:1787827173.145985:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421022 of 2048 to 1024 bytes 00000080:00200000:3.0:1787827173.145988:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724508160, len 1024, copied 1024 00000080:00200000:3.0:1787827173.145991:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724509184 i_blocks 3365288 00000080:00200000:3.0:1787827173.145994:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.145998:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724509184, count: 1024 00000080:00200000:3.0:1787827173.146001:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421022 of 3072 to 1024 bytes 00000080:00200000:3.0:1787827173.146004:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724509184, len 1024, copied 1024 00000080:00200000:3.0:1787827173.146008:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724510208 i_blocks 3365288 00000080:00200000:3.0:1787827173.146010:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.146014:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724510208, count: 1024 00000080:00200000:3.0:1787827173.146017:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421023 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.146019:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 0, original count 1024 00000080:00200000:3.0:1787827173.146022:0:27439:0:(file.c:2090:ll_file_io_generic()) oosfile: write ppos: 1724510208, bytes: 1024 00000080:00200000:3.0:1787827173.146024:0:27439:0:(file.c:1848:ll_io_set_mirror()) oosfile: desiginated mirror: 0 00000020:00200000:3.0:1787827173.146028:0:27439:0:(cl_io.c:218:cl_io_rw_init()) header@000000007788600a[0x110, 368576, [0x200000402:0x27:0x0] hash] 00000020:00200000:3.0:1787827173.146030:0:27439:0:(cl_io.c:218:cl_io_rw_init()) io range: 2 [1724510208, 1724511232) 0 0 00000080:00200000:3.0:1787827173.146032:0:27439:0:(vvp_io.c:1889:vvp_io_init()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1787827173.146038:0:27439:0:(file.c:2170:ll_file_io_generic()) Range lock [421023, 421023] 00020000:00200000:3.0:1787827173.146041:0:27439:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 411 chunk: [1724510208, 1724511232) 1724511232, 1024 00020000:00200000:3.0:1787827173.146044:0:27439:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1787827173.146049:0:27439:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [421023, 421023] 00000008:00010000:3.0:1787827173.146054:0:27439:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1787827173.146057:0:27439:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff9e35c7d81d30, osc lock ffff9e35c2ddcc30, flags 0x0 00010000:00010000:3.0:1787827173.146065:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.146075:0:27439:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (860483584 860487679) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.146084:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 20/0,2 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.146092:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 20/0,2 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.146100:0:27439:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 19/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.146109:0:27439:0:(vvp_io.c:1339:vvp_io_write_start()) oosfile: write [1724510208, 1724511232) append 0 append lockpos 0 00000080:00200000:3.0:1787827173.146115:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421023 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.146132:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724510208, len 1024, copied 1024 00000080:00200000:3.0:1787827173.146137:0:27439:0:(rw26.c:933:ll_write_end()) page@000000005db1dca8[3 000000005ce0a915 2 1 00000000a23cbd2a 1 0 421023] 00000080:00200000:3.0:1787827173.146140:0:27439:0:(rw26.c:933:ll_write_end()) vmpage @0000000069aa2ff6 fffffc0001001 3:0 ffff9e34c3cb1000 421023 lru 00000080:00200000:3.0:1787827173.146145:0:27439:0:(rw26.c:933:ll_write_end()) osc-page@00000000d53ab8a5 210079: 1< 0 - - > 2< 860483584 0 0 0x0 0x0 | 000000004db7037e 0000000052729811 > 3< 0 0 > 4< 0 0 8 483962880 - | - - + - > 5< - - + - | 0 - | 377 - -> 00000080:00200000:3.0:1787827173.146147:0:27439:0:(rw26.c:933:ll_write_end()) end page@000000005db1dca8 00000080:00200000:3.0:1787827173.146148:0:27439:0:(rw26.c:933:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1787827173.146150:0:27439:0:(vvp_io.c:1214:vvp_io_write_commit()) commit async pages: 1, from 0, to 1024 prio 0 00000008:00010000:3.0:1787827173.146159:0:27439:0:(osc_cache.c:565:osc_extent_merge()) ### extent: 000000001063fa34 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.146173:0:27439:0:(osc_cache.c:497:__osc_extent_remove()) ### extent: 000000001063fa34 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.146180:0:27439:0:(osc_cache.c:582:osc_extent_merge()) ### extent: 00000000df269729 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.146189:0:27439:0:(osc_cache.c:846:osc_extent_find()) ### extent: 00000000df269729 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.146202:0:27439:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff9e35d0e392f8, count 1, dirtied 1 00000080:00200000:3.0:1787827173.146205:0:27439:0:(vvp_io.c:1237:vvp_io_write_commit()) Committed 1 pages 1024 bytes, tot: 1024 00000080:00200000:3.0:1787827173.146207:0:27439:0:(vvp_io.c:1421:vvp_io_write_start()) oosfile: write bytes 0, result: 1024 00000020:00010000:3.0:1787827173.146214:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_release():245 00000020:00010000:3.0:1787827173.146216:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1787827173.146219:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c7d81d30 (ffff9e35da704cf8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1787827173.146224:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.146232:0:27439:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 17/0,0 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1787827173.146242:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_fini():72 00000020:00010000:3.0:1787827173.146245:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c7d81d30 (ffff9e35c614ce18/1) at cl_lock_fini():72 00000080:00200000:3.0:1787827173.146249:0:27439:0:(vvp_io.c:533:vvp_io_advance()) advancing 1024 bytes 00000080:00200000:3.0:1787827173.146252:0:27439:0:(file.c:2207:ll_file_io_generic()) Range unlock [421023, 421023] 00000080:00200000:3.0:1787827173.146256:0:27439:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787827173.146259:0:27439:0:(file.c:2240:ll_file_io_generic()) oosfile: 2 io complete with rc: 0, result: 1024, restart: 0 00000080:00200000:3.0:1787827173.146264:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724511232 i_blocks 3365288 00000080:00200000:3.0:1787827173.146267:0:27439:0:(file.c:2326:ll_file_io_generic()) iot: 2, result: 1024 00000080:00200008:3.0:1787827173.146274:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724511232, count: 1024 00000080:00200000:3.0:1787827173.146278:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421023 of 1024 to 1024 bytes 00000080:00200000:3.0:1787827173.146282:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724511232, len 1024, copied 1024 00000080:00200000:3.0:1787827173.146286:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724512256 i_blocks 3365288 00000080:00200000:3.0:1787827173.146288:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.146292:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724512256, count: 1024 00000080:00200000:3.0:1787827173.146295:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421023 of 2048 to 1024 bytes 00000080:00200000:3.0:1787827173.146297:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724512256, len 1024, copied 1024 00000080:00200000:3.0:1787827173.146301:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724513280 i_blocks 3365288 00000080:00200000:3.0:1787827173.146302:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.146305:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724513280, count: 1024 00000080:00200000:3.0:1787827173.146307:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421023 of 3072 to 1024 bytes 00000080:00200000:3.0:1787827173.146308:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724513280, len 1024, copied 1024 00000080:00200000:3.0:1787827173.146310:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724514304 i_blocks 3365288 00000080:00200000:3.0:1787827173.146312:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.146314:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724514304, count: 1024 00000080:00200000:3.0:1787827173.146316:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421024 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.146317:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 0, original count 1024 00000080:00200000:3.0:1787827173.146318:0:27439:0:(file.c:2090:ll_file_io_generic()) oosfile: write ppos: 1724514304, bytes: 1024 00000080:00200000:3.0:1787827173.146320:0:27439:0:(file.c:1848:ll_io_set_mirror()) oosfile: desiginated mirror: 0 00000020:00200000:3.0:1787827173.146322:0:27439:0:(cl_io.c:218:cl_io_rw_init()) header@000000007788600a[0x110, 368577, [0x200000402:0x27:0x0] hash] 00000020:00200000:3.0:1787827173.146323:0:27439:0:(cl_io.c:218:cl_io_rw_init()) io range: 2 [1724514304, 1724515328) 0 0 00000080:00200000:3.0:1787827173.146324:0:27439:0:(vvp_io.c:1889:vvp_io_init()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1787827173.146328:0:27439:0:(file.c:2170:ll_file_io_generic()) Range lock [421024, 421024] 00020000:00200000:3.0:1787827173.146331:0:27439:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 411 chunk: [1724514304, 1724515328) 1724515328, 1024 00020000:00200000:3.0:1787827173.146333:0:27439:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1787827173.146338:0:27439:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [421024, 421024] 00000008:00010000:3.0:1787827173.146344:0:27439:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1787827173.146346:0:27439:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff9e35c7d81d30, osc lock ffff9e35c2ddcc30, flags 0x0 00010000:00010000:3.0:1787827173.146354:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.146363:0:27439:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (860487680 860491775) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.146372:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 20/0,2 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.146381:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 20/0,2 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.146388:0:27439:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 19/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.146395:0:27439:0:(vvp_io.c:1339:vvp_io_write_start()) oosfile: write [1724514304, 1724515328) append 0 append lockpos 0 00000080:00200000:3.0:1787827173.146399:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421024 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.146412:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724514304, len 1024, copied 1024 00000080:00200000:3.0:1787827173.146416:0:27439:0:(rw26.c:933:ll_write_end()) page@000000002c881f79[3 000000005ce0a915 2 1 00000000a23cbd2a 1 0 421024] 00000080:00200000:3.0:1787827173.146418:0:27439:0:(rw26.c:933:ll_write_end()) vmpage @0000000091f5fdae fffffc0001001 3:0 ffff9e34c3cb12b8 421024 lru 00000080:00200000:3.0:1787827173.146422:0:27439:0:(rw26.c:933:ll_write_end()) osc-page@000000008ea6b22f 210080: 1< 0 - - > 2< 860487680 0 0 0x0 0x0 | 000000004db7037e 0000000052729811 > 3< 0 0 > 4< 0 0 8 483958784 - | - - + - > 5< - - + - | 0 - | 378 - -> 00000080:00200000:3.0:1787827173.146423:0:27439:0:(rw26.c:933:ll_write_end()) end page@000000002c881f79 00000080:00200000:3.0:1787827173.146424:0:27439:0:(rw26.c:933:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1787827173.146425:0:27439:0:(vvp_io.c:1214:vvp_io_write_commit()) commit async pages: 1, from 0, to 1024 prio 0 00000008:00010000:3.0:1787827173.146431:0:27439:0:(osc_cache.c:565:osc_extent_merge()) ### extent: 000000001063fa34 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.146443:0:27439:0:(osc_cache.c:497:__osc_extent_remove()) ### extent: 000000001063fa34 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.146450:0:27439:0:(osc_cache.c:582:osc_extent_merge()) ### extent: 00000000df269729 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.146457:0:27439:0:(osc_cache.c:846:osc_extent_find()) ### extent: 00000000df269729 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.146469:0:27439:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff9e35d0e392f8, count 1, dirtied 1 00000080:00200000:3.0:1787827173.146472:0:27439:0:(vvp_io.c:1237:vvp_io_write_commit()) Committed 1 pages 1024 bytes, tot: 1024 00000080:00200000:3.0:1787827173.146474:0:27439:0:(vvp_io.c:1421:vvp_io_write_start()) oosfile: write bytes 0, result: 1024 00000020:00010000:3.0:1787827173.146480:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_release():245 00000020:00010000:3.0:1787827173.146483:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1787827173.146485:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c7d81d30 (ffff9e35da704cf8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1787827173.146490:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.146495:0:27439:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 17/0,0 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1787827173.146501:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_fini():72 00000020:00010000:3.0:1787827173.146502:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c7d81d30 (ffff9e35c614ce18/1) at cl_lock_fini():72 00000080:00200000:3.0:1787827173.146505:0:27439:0:(vvp_io.c:533:vvp_io_advance()) advancing 1024 bytes 00000080:00200000:3.0:1787827173.146507:0:27439:0:(file.c:2207:ll_file_io_generic()) Range unlock [421024, 421024] 00000080:00200000:3.0:1787827173.146510:0:27439:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787827173.146512:0:27439:0:(file.c:2240:ll_file_io_generic()) oosfile: 2 io complete with rc: 0, result: 1024, restart: 0 00000080:00200000:3.0:1787827173.146516:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724515328 i_blocks 3365288 00000080:00200000:3.0:1787827173.146517:0:27439:0:(file.c:2326:ll_file_io_generic()) iot: 2, result: 1024 00000080:00200008:3.0:1787827173.146523:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724515328, count: 1024 00000080:00200000:3.0:1787827173.146526:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421024 of 1024 to 1024 bytes 00000080:00200000:3.0:1787827173.146529:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724515328, len 1024, copied 1024 00000080:00200000:3.0:1787827173.146532:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724516352 i_blocks 3365288 00000080:00200000:3.0:1787827173.146534:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.146536:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724516352, count: 1024 00000080:00200000:3.0:1787827173.146538:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421024 of 2048 to 1024 bytes 00000080:00200000:3.0:1787827173.146539:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724516352, len 1024, copied 1024 00000080:00200000:3.0:1787827173.146541:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724517376 i_blocks 3365288 00000080:00200000:3.0:1787827173.146543:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.146545:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724517376, count: 1024 00000080:00200000:3.0:1787827173.146546:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421024 of 3072 to 1024 bytes 00000080:00200000:3.0:1787827173.146548:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724517376, len 1024, copied 1024 00000080:00200000:3.0:1787827173.146550:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724518400 i_blocks 3365288 00000080:00200000:3.0:1787827173.146551:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.146553:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724518400, count: 1024 00000080:00200000:3.0:1787827173.146555:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421025 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.146556:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 0, original count 1024 00000080:00200000:3.0:1787827173.146557:0:27439:0:(file.c:2090:ll_file_io_generic()) oosfile: write ppos: 1724518400, bytes: 1024 00000080:00200000:3.0:1787827173.146558:0:27439:0:(file.c:1848:ll_io_set_mirror()) oosfile: desiginated mirror: 0 00000020:00200000:3.0:1787827173.146560:0:27439:0:(cl_io.c:218:cl_io_rw_init()) header@000000007788600a[0x110, 368578, [0x200000402:0x27:0x0] hash] 00000020:00200000:3.0:1787827173.146561:0:27439:0:(cl_io.c:218:cl_io_rw_init()) io range: 2 [1724518400, 1724519424) 0 0 00000080:00200000:3.0:1787827173.146563:0:27439:0:(vvp_io.c:1889:vvp_io_init()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1787827173.146566:0:27439:0:(file.c:2170:ll_file_io_generic()) Range lock [421025, 421025] 00020000:00200000:3.0:1787827173.146568:0:27439:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 411 chunk: [1724518400, 1724519424) 1724519424, 1024 00020000:00200000:3.0:1787827173.146569:0:27439:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1787827173.146572:0:27439:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [421025, 421025] 00000008:00010000:3.0:1787827173.146576:0:27439:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1787827173.146578:0:27439:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff9e35c7d81d30, osc lock ffff9e35c2ddcc30, flags 0x0 00010000:00010000:3.0:1787827173.146583:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.146589:0:27439:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (860491776 860495871) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.146594:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 20/0,2 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.146599:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 20/0,2 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.146603:0:27439:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 19/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.146608:0:27439:0:(vvp_io.c:1339:vvp_io_write_start()) oosfile: write [1724518400, 1724519424) append 0 append lockpos 0 00000080:00200000:3.0:1787827173.146611:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421025 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.146623:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724518400, len 1024, copied 1024 00000080:00200000:3.0:1787827173.146627:0:27439:0:(rw26.c:933:ll_write_end()) page@00000000da71ae43[3 000000005ce0a915 2 1 00000000a23cbd2a 1 0 421025] 00000080:00200000:3.0:1787827173.146629:0:27439:0:(rw26.c:933:ll_write_end()) vmpage @00000000e505b705 fffffc0001001 3:0 ffff9e34c3cb1488 421025 lru 00000080:00200000:3.0:1787827173.146632:0:27439:0:(rw26.c:933:ll_write_end()) osc-page@0000000086bf7d2c 210081: 1< 0 - - > 2< 860491776 0 0 0x0 0x0 | 000000004db7037e 0000000052729811 > 3< 0 0 > 4< 0 0 8 483954688 - | - - + - > 5< - - + - | 0 - | 379 - -> 00000080:00200000:3.0:1787827173.146634:0:27439:0:(rw26.c:933:ll_write_end()) end page@00000000da71ae43 00000080:00200000:3.0:1787827173.146634:0:27439:0:(rw26.c:933:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1787827173.146636:0:27439:0:(vvp_io.c:1214:vvp_io_write_commit()) commit async pages: 1, from 0, to 1024 prio 0 00000008:00010000:3.0:1787827173.146641:0:27439:0:(osc_cache.c:565:osc_extent_merge()) ### extent: 000000001063fa34 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.146650:0:27439:0:(osc_cache.c:497:__osc_extent_remove()) ### extent: 000000001063fa34 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.146654:0:27439:0:(osc_cache.c:582:osc_extent_merge()) ### extent: 00000000df269729 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.146658:0:27439:0:(osc_cache.c:846:osc_extent_find()) ### extent: 00000000df269729 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.146664:0:27439:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff9e35d0e392f8, count 1, dirtied 1 00000080:00200000:3.0:1787827173.146666:0:27439:0:(vvp_io.c:1237:vvp_io_write_commit()) Committed 1 pages 1024 bytes, tot: 1024 00000080:00200000:3.0:1787827173.146667:0:27439:0:(vvp_io.c:1421:vvp_io_write_start()) oosfile: write bytes 0, result: 1024 00000020:00010000:3.0:1787827173.146671:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_release():245 00000020:00010000:3.0:1787827173.146672:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1787827173.146674:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c7d81d30 (ffff9e35da704cf8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1787827173.146676:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.146681:0:27439:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 17/0,0 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1787827173.146686:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_fini():72 00000020:00010000:3.0:1787827173.146688:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c7d81d30 (ffff9e35c614ce18/1) at cl_lock_fini():72 00000080:00200000:3.0:1787827173.146690:0:27439:0:(vvp_io.c:533:vvp_io_advance()) advancing 1024 bytes 00000080:00200000:3.0:1787827173.146692:0:27439:0:(file.c:2207:ll_file_io_generic()) Range unlock [421025, 421025] 00000080:00200000:3.0:1787827173.146695:0:27439:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787827173.146697:0:27439:0:(file.c:2240:ll_file_io_generic()) oosfile: 2 io complete with rc: 0, result: 1024, restart: 0 00000080:00200000:3.0:1787827173.146700:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724519424 i_blocks 3365288 00000080:00200000:3.0:1787827173.146701:0:27439:0:(file.c:2326:ll_file_io_generic()) iot: 2, result: 1024 00000080:00200008:3.0:1787827173.146707:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724519424, count: 1024 00000080:00200000:3.0:1787827173.146710:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421025 of 1024 to 1024 bytes 00000080:00200000:3.0:1787827173.146711:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724519424, len 1024, copied 1024 00000080:00200000:3.0:1787827173.146714:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724520448 i_blocks 3365288 00000080:00200000:3.0:1787827173.146716:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.146718:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724520448, count: 1024 00000080:00200000:3.0:1787827173.146720:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421025 of 2048 to 1024 bytes 00000080:00200000:3.0:1787827173.146721:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724520448, len 1024, copied 1024 00000080:00200000:3.0:1787827173.146723:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724521472 i_blocks 3365288 00000080:00200000:3.0:1787827173.146725:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.146727:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724521472, count: 1024 00000080:00200000:3.0:1787827173.146728:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421025 of 3072 to 1024 bytes 00000080:00200000:3.0:1787827173.146729:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724521472, len 1024, copied 1024 00000080:00200000:3.0:1787827173.146731:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724522496 i_blocks 3365288 00000080:00200000:3.0:1787827173.146733:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.146734:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724522496, count: 1024 00000080:00200000:3.0:1787827173.146736:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421026 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.146737:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 0, original count 1024 00000080:00200000:3.0:1787827173.146738:0:27439:0:(file.c:2090:ll_file_io_generic()) oosfile: write ppos: 1724522496, bytes: 1024 00000080:00200000:3.0:1787827173.146740:0:27439:0:(file.c:1848:ll_io_set_mirror()) oosfile: desiginated mirror: 0 00000020:00200000:3.0:1787827173.146742:0:27439:0:(cl_io.c:218:cl_io_rw_init()) header@000000007788600a[0x110, 368579, [0x200000402:0x27:0x0] hash] 00000020:00200000:3.0:1787827173.146743:0:27439:0:(cl_io.c:218:cl_io_rw_init()) io range: 2 [1724522496, 1724523520) 0 0 00000080:00200000:3.0:1787827173.146744:0:27439:0:(vvp_io.c:1889:vvp_io_init()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1787827173.146748:0:27439:0:(file.c:2170:ll_file_io_generic()) Range lock [421026, 421026] 00020000:00200000:3.0:1787827173.146750:0:27439:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 411 chunk: [1724522496, 1724523520) 1724523520, 1024 00020000:00200000:3.0:1787827173.146751:0:27439:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1787827173.146754:0:27439:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [421026, 421026] 00000008:00010000:3.0:1787827173.146758:0:27439:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1787827173.146760:0:27439:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff9e35c7d81d30, osc lock ffff9e35c2ddcc30, flags 0x0 00010000:00010000:3.0:1787827173.146766:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.146774:0:27439:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (860495872 860499967) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.146783:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 20/0,2 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.146791:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 20/0,2 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.146823:0:27439:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 19/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.146832:0:27439:0:(vvp_io.c:1339:vvp_io_write_start()) oosfile: write [1724522496, 1724523520) append 0 append lockpos 0 00000080:00200000:3.0:1787827173.146838:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421026 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.146861:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724522496, len 1024, copied 1024 00000080:00200000:3.0:1787827173.146866:0:27439:0:(rw26.c:933:ll_write_end()) page@00000000c752d97b[3 000000005ce0a915 2 1 00000000a23cbd2a 1 0 421026] 00000080:00200000:3.0:1787827173.146868:0:27439:0:(rw26.c:933:ll_write_end()) vmpage @00000000fdfd9ed0 fffffc0001021 2:0 ffff9e34c3cb1bc8 421026 lru 00000080:00200000:3.0:1787827173.146872:0:27439:0:(rw26.c:933:ll_write_end()) osc-page@000000007be71385 210082: 1< 0 - - > 2< 860495872 0 0 0x0 0x0 | 000000004db7037e 0000000052729811 > 3< 0 0 > 4< 0 0 8 483950592 - | - - + - > 5< - - + - | 0 - | 380 - -> 00000080:00200000:3.0:1787827173.146873:0:27439:0:(rw26.c:933:ll_write_end()) end page@00000000c752d97b 00000080:00200000:3.0:1787827173.146874:0:27439:0:(rw26.c:933:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1787827173.146876:0:27439:0:(vvp_io.c:1214:vvp_io_write_commit()) commit async pages: 1, from 0, to 1024 prio 0 00000008:00010000:3.0:1787827173.146882:0:27439:0:(osc_cache.c:565:osc_extent_merge()) ### extent: 000000001063fa34 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.146895:0:27439:0:(osc_cache.c:497:__osc_extent_remove()) ### extent: 000000001063fa34 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.146902:0:27439:0:(osc_cache.c:582:osc_extent_merge()) ### extent: 00000000df269729 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.146910:0:27439:0:(osc_cache.c:846:osc_extent_find()) ### extent: 00000000df269729 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.146933:0:27439:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff9e35d0e392f8, count 1, dirtied 1 00000080:00200000:3.0:1787827173.146936:0:27439:0:(vvp_io.c:1237:vvp_io_write_commit()) Committed 1 pages 1024 bytes, tot: 1024 00000080:00200000:3.0:1787827173.146939:0:27439:0:(vvp_io.c:1421:vvp_io_write_start()) oosfile: write bytes 0, result: 1024 00000020:00010000:3.0:1787827173.146945:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_release():245 00000020:00010000:3.0:1787827173.146947:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1787827173.146949:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c7d81d30 (ffff9e35da704cf8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1787827173.146952:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.146958:0:27439:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 17/0,0 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1787827173.146963:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_fini():72 00000020:00010000:3.0:1787827173.146965:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c7d81d30 (ffff9e35c614ce18/1) at cl_lock_fini():72 00000080:00200000:3.0:1787827173.146967:0:27439:0:(vvp_io.c:533:vvp_io_advance()) advancing 1024 bytes 00000080:00200000:3.0:1787827173.146969:0:27439:0:(file.c:2207:ll_file_io_generic()) Range unlock [421026, 421026] 00000080:00200000:3.0:1787827173.146972:0:27439:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787827173.146974:0:27439:0:(file.c:2240:ll_file_io_generic()) oosfile: 2 io complete with rc: 0, result: 1024, restart: 0 00000080:00200000:3.0:1787827173.146977:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724523520 i_blocks 3365288 00000080:00200000:3.0:1787827173.146979:0:27439:0:(file.c:2326:ll_file_io_generic()) iot: 2, result: 1024 00000080:00200008:3.0:1787827173.146985:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724523520, count: 1024 00000080:00200000:3.0:1787827173.146988:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421026 of 1024 to 1024 bytes 00000080:00200000:3.0:1787827173.146990:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724523520, len 1024, copied 1024 00000080:00200000:3.0:1787827173.146993:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724524544 i_blocks 3365288 00000080:00200000:3.0:1787827173.146995:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.146997:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724524544, count: 1024 00000080:00200000:3.0:1787827173.146999:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421026 of 2048 to 1024 bytes 00000080:00200000:3.0:1787827173.147000:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724524544, len 1024, copied 1024 00000080:00200000:3.0:1787827173.147002:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724525568 i_blocks 3365288 00000080:00200000:3.0:1787827173.147004:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.147006:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724525568, count: 1024 00000080:00200000:3.0:1787827173.147008:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421026 of 3072 to 1024 bytes 00000080:00200000:3.0:1787827173.147009:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724525568, len 1024, copied 1024 00000080:00200000:3.0:1787827173.147011:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724526592 i_blocks 3365288 00000080:00200000:3.0:1787827173.147012:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.147014:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724526592, count: 1024 00000080:00200000:3.0:1787827173.147016:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421027 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.147017:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 0, original count 1024 00000080:00200000:3.0:1787827173.147018:0:27439:0:(file.c:2090:ll_file_io_generic()) oosfile: write ppos: 1724526592, bytes: 1024 00000080:00200000:3.0:1787827173.147019:0:27439:0:(file.c:1848:ll_io_set_mirror()) oosfile: desiginated mirror: 0 00000020:00200000:3.0:1787827173.147021:0:27439:0:(cl_io.c:218:cl_io_rw_init()) header@000000007788600a[0x110, 368580, [0x200000402:0x27:0x0] hash] 00000020:00200000:3.0:1787827173.147022:0:27439:0:(cl_io.c:218:cl_io_rw_init()) io range: 2 [1724526592, 1724527616) 0 0 00000080:00200000:3.0:1787827173.147024:0:27439:0:(vvp_io.c:1889:vvp_io_init()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1787827173.147027:0:27439:0:(file.c:2170:ll_file_io_generic()) Range lock [421027, 421027] 00020000:00200000:3.0:1787827173.147029:0:27439:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 411 chunk: [1724526592, 1724527616) 1724527616, 1024 00020000:00200000:3.0:1787827173.147030:0:27439:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1787827173.147033:0:27439:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [421027, 421027] 00000008:00010000:3.0:1787827173.147037:0:27439:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1787827173.147039:0:27439:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff9e35c7d81d30, osc lock ffff9e35c2ddcc30, flags 0x0 00010000:00010000:3.0:1787827173.147044:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.147050:0:27439:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (860499968 860504063) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.147056:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 20/0,2 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.147060:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 20/0,2 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.147064:0:27439:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 19/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.147069:0:27439:0:(vvp_io.c:1339:vvp_io_write_start()) oosfile: write [1724526592, 1724527616) append 0 append lockpos 0 00000080:00200000:3.0:1787827173.147072:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421027 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.147088:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724526592, len 1024, copied 1024 00000080:00200000:3.0:1787827173.147093:0:27439:0:(rw26.c:933:ll_write_end()) page@000000002d069805[3 000000005ce0a915 2 1 00000000a23cbd2a 1 0 421027] 00000080:00200000:3.0:1787827173.147096:0:27439:0:(rw26.c:933:ll_write_end()) vmpage @00000000a5323382 fffffc0001001 3:0 ffff9e34c3cb19f8 421027 lru 00000080:00200000:3.0:1787827173.147101:0:27439:0:(rw26.c:933:ll_write_end()) osc-page@00000000e97a0271 210083: 1< 0 - - > 2< 860499968 0 0 0x0 0x0 | 000000004db7037e 0000000052729811 > 3< 0 0 > 4< 0 0 8 483946496 - | - - + - > 5< - - + - | 0 - | 381 - -> 00000080:00200000:3.0:1787827173.147103:0:27439:0:(rw26.c:933:ll_write_end()) end page@000000002d069805 00000080:00200000:3.0:1787827173.147105:0:27439:0:(rw26.c:933:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1787827173.147107:0:27439:0:(vvp_io.c:1214:vvp_io_write_commit()) commit async pages: 1, from 0, to 1024 prio 0 00000008:00010000:3.0:1787827173.147115:0:27439:0:(osc_cache.c:565:osc_extent_merge()) ### extent: 000000001063fa34 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.147129:0:27439:0:(osc_cache.c:497:__osc_extent_remove()) ### extent: 000000001063fa34 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.147137:0:27439:0:(osc_cache.c:582:osc_extent_merge()) ### extent: 00000000df269729 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.147144:0:27439:0:(osc_cache.c:846:osc_extent_find()) ### extent: 00000000df269729 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.147155:0:27439:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff9e35d0e392f8, count 1, dirtied 1 00000080:00200000:3.0:1787827173.147159:0:27439:0:(vvp_io.c:1237:vvp_io_write_commit()) Committed 1 pages 1024 bytes, tot: 1024 00000080:00200000:3.0:1787827173.147161:0:27439:0:(vvp_io.c:1421:vvp_io_write_start()) oosfile: write bytes 0, result: 1024 00000020:00010000:3.0:1787827173.147167:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_release():245 00000020:00010000:3.0:1787827173.147170:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1787827173.147172:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c7d81d30 (ffff9e35da704cf8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1787827173.147175:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.147181:0:27439:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 17/0,0 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1787827173.147187:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_fini():72 00000020:00010000:3.0:1787827173.147188:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c7d81d30 (ffff9e35c614ce18/1) at cl_lock_fini():72 00000080:00200000:3.0:1787827173.147191:0:27439:0:(vvp_io.c:533:vvp_io_advance()) advancing 1024 bytes 00000080:00200000:3.0:1787827173.147193:0:27439:0:(file.c:2207:ll_file_io_generic()) Range unlock [421027, 421027] 00000080:00200000:3.0:1787827173.147196:0:27439:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787827173.147198:0:27439:0:(file.c:2240:ll_file_io_generic()) oosfile: 2 io complete with rc: 0, result: 1024, restart: 0 00000080:00200000:3.0:1787827173.147202:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724527616 i_blocks 3365288 00000080:00200000:3.0:1787827173.147203:0:27439:0:(file.c:2326:ll_file_io_generic()) iot: 2, result: 1024 00000080:00200008:3.0:1787827173.147209:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724527616, count: 1024 00000080:00200000:3.0:1787827173.147212:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421027 of 1024 to 1024 bytes 00000080:00200000:3.0:1787827173.147214:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724527616, len 1024, copied 1024 00000080:00200000:3.0:1787827173.147217:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724528640 i_blocks 3365288 00000080:00200000:3.0:1787827173.147219:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.147221:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724528640, count: 1024 00000080:00200000:3.0:1787827173.147223:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421027 of 2048 to 1024 bytes 00000080:00200000:3.0:1787827173.147224:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724528640, len 1024, copied 1024 00000080:00200000:3.0:1787827173.147226:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724529664 i_blocks 3365288 00000080:00200000:3.0:1787827173.147228:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.147230:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724529664, count: 1024 00000080:00200000:3.0:1787827173.147234:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421027 of 3072 to 1024 bytes 00000080:00200000:3.0:1787827173.147236:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724529664, len 1024, copied 1024 00000080:00200000:3.0:1787827173.147240:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724530688 i_blocks 3365288 00000080:00200000:3.0:1787827173.147242:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.147246:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724530688, count: 1024 00000080:00200000:3.0:1787827173.147249:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421028 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.147252:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 0, original count 1024 00000080:00200000:3.0:1787827173.147253:0:27439:0:(file.c:2090:ll_file_io_generic()) oosfile: write ppos: 1724530688, bytes: 1024 00000080:00200000:3.0:1787827173.147256:0:27439:0:(file.c:1848:ll_io_set_mirror()) oosfile: desiginated mirror: 0 00000020:00200000:3.0:1787827173.147259:0:27439:0:(cl_io.c:218:cl_io_rw_init()) header@000000007788600a[0x110, 368581, [0x200000402:0x27:0x0] hash] 00000020:00200000:3.0:1787827173.147261:0:27439:0:(cl_io.c:218:cl_io_rw_init()) io range: 2 [1724530688, 1724531712) 0 0 00000080:00200000:3.0:1787827173.147264:0:27439:0:(vvp_io.c:1889:vvp_io_init()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1787827173.147269:0:27439:0:(file.c:2170:ll_file_io_generic()) Range lock [421028, 421028] 00020000:00200000:3.0:1787827173.147272:0:27439:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 411 chunk: [1724530688, 1724531712) 1724531712, 1024 00020000:00200000:3.0:1787827173.147275:0:27439:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1787827173.147279:0:27439:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [421028, 421028] 00000008:00010000:3.0:1787827173.147285:0:27439:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1787827173.147287:0:27439:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff9e35c7d81d30, osc lock ffff9e35c2ddcc30, flags 0x0 00010000:00010000:3.0:1787827173.147294:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.147300:0:27439:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (860504064 860508159) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.147306:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 20/0,2 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.147311:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 20/0,2 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.147315:0:27439:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 19/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.147320:0:27439:0:(vvp_io.c:1339:vvp_io_write_start()) oosfile: write [1724530688, 1724531712) append 0 append lockpos 0 00000080:00200000:3.0:1787827173.147323:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421028 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.147338:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724530688, len 1024, copied 1024 00000080:00200000:3.0:1787827173.147343:0:27439:0:(rw26.c:933:ll_write_end()) page@00000000a34a7e13[3 000000005ce0a915 2 1 00000000a23cbd2a 1 0 421028] 00000080:00200000:3.0:1787827173.147346:0:27439:0:(rw26.c:933:ll_write_end()) vmpage @00000000db66da61 fffffc0001001 3:0 ffff9e34c3cb1ae0 421028 lru 00000080:00200000:3.0:1787827173.147351:0:27439:0:(rw26.c:933:ll_write_end()) osc-page@00000000436cdd3c 210084: 1< 0 - - > 2< 860504064 0 0 0x0 0x0 | 000000004db7037e 0000000052729811 > 3< 0 0 > 4< 0 0 8 483942400 - | - - + - > 5< - - + - | 0 - | 382 - -> 00000080:00200000:3.0:1787827173.147353:0:27439:0:(rw26.c:933:ll_write_end()) end page@00000000a34a7e13 00000080:00200000:3.0:1787827173.147354:0:27439:0:(rw26.c:933:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1787827173.147356:0:27439:0:(vvp_io.c:1214:vvp_io_write_commit()) commit async pages: 1, from 0, to 1024 prio 0 00000008:00010000:3.0:1787827173.147365:0:27439:0:(osc_cache.c:565:osc_extent_merge()) ### extent: 000000001063fa34 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.147378:0:27439:0:(osc_cache.c:497:__osc_extent_remove()) ### extent: 000000001063fa34 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.147385:0:27439:0:(osc_cache.c:582:osc_extent_merge()) ### extent: 00000000df269729 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.147392:0:27439:0:(osc_cache.c:846:osc_extent_find()) ### extent: 00000000df269729 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.147402:0:27439:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff9e35d0e392f8, count 1, dirtied 1 00000080:00200000:3.0:1787827173.147405:0:27439:0:(vvp_io.c:1237:vvp_io_write_commit()) Committed 1 pages 1024 bytes, tot: 1024 00000080:00200000:3.0:1787827173.147407:0:27439:0:(vvp_io.c:1421:vvp_io_write_start()) oosfile: write bytes 0, result: 1024 00000020:00010000:3.0:1787827173.147413:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_release():245 00000020:00010000:3.0:1787827173.147415:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1787827173.147418:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c7d81d30 (ffff9e35da704cf8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1787827173.147422:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.147430:0:27439:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 17/0,0 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1787827173.147439:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_fini():72 00000020:00010000:3.0:1787827173.147442:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c7d81d30 (ffff9e35c614ce18/1) at cl_lock_fini():72 00000080:00200000:3.0:1787827173.147446:0:27439:0:(vvp_io.c:533:vvp_io_advance()) advancing 1024 bytes 00000080:00200000:3.0:1787827173.147449:0:27439:0:(file.c:2207:ll_file_io_generic()) Range unlock [421028, 421028] 00000080:00200000:3.0:1787827173.147453:0:27439:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787827173.147456:0:27439:0:(file.c:2240:ll_file_io_generic()) oosfile: 2 io complete with rc: 0, result: 1024, restart: 0 00000080:00200000:3.0:1787827173.147460:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724531712 i_blocks 3365288 00000080:00200000:3.0:1787827173.147463:0:27439:0:(file.c:2326:ll_file_io_generic()) iot: 2, result: 1024 00000080:00200008:3.0:1787827173.147469:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724531712, count: 1024 00000080:00200000:3.0:1787827173.147473:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421028 of 1024 to 1024 bytes 00000080:00200000:3.0:1787827173.147476:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724531712, len 1024, copied 1024 00000080:00200000:3.0:1787827173.147480:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724532736 i_blocks 3365288 00000080:00200000:3.0:1787827173.147483:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.147486:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724532736, count: 1024 00000080:00200000:3.0:1787827173.147489:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421028 of 2048 to 1024 bytes 00000080:00200000:3.0:1787827173.147491:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724532736, len 1024, copied 1024 00000080:00200000:3.0:1787827173.147494:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724533760 i_blocks 3365288 00000080:00200000:3.0:1787827173.147497:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.147500:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724533760, count: 1024 00000080:00200000:3.0:1787827173.147503:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421028 of 3072 to 1024 bytes 00000080:00200000:3.0:1787827173.147506:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724533760, len 1024, copied 1024 00000080:00200000:3.0:1787827173.147509:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724534784 i_blocks 3365288 00000080:00200000:3.0:1787827173.147511:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.147515:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724534784, count: 1024 00000080:00200000:3.0:1787827173.147518:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421029 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.147520:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 0, original count 1024 00000080:00200000:3.0:1787827173.147521:0:27439:0:(file.c:2090:ll_file_io_generic()) oosfile: write ppos: 1724534784, bytes: 1024 00000080:00200000:3.0:1787827173.147523:0:27439:0:(file.c:1848:ll_io_set_mirror()) oosfile: desiginated mirror: 0 00000020:00200000:3.0:1787827173.147526:0:27439:0:(cl_io.c:218:cl_io_rw_init()) header@000000007788600a[0x110, 368582, [0x200000402:0x27:0x0] hash] 00000020:00200000:3.0:1787827173.147528:0:27439:0:(cl_io.c:218:cl_io_rw_init()) io range: 2 [1724534784, 1724535808) 0 0 00000080:00200000:3.0:1787827173.147531:0:27439:0:(vvp_io.c:1889:vvp_io_init()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1787827173.147536:0:27439:0:(file.c:2170:ll_file_io_generic()) Range lock [421029, 421029] 00020000:00200000:3.0:1787827173.147539:0:27439:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 411 chunk: [1724534784, 1724535808) 1724535808, 1024 00020000:00200000:3.0:1787827173.147541:0:27439:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1787827173.147546:0:27439:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [421029, 421029] 00000008:00010000:3.0:1787827173.147551:0:27439:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1787827173.147554:0:27439:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff9e35c7d81d30, osc lock ffff9e35c2ddcc30, flags 0x0 00010000:00010000:3.0:1787827173.147562:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.147571:0:27439:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (860508160 860512255) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.147579:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 20/0,2 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.147587:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 20/0,2 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.147594:0:27439:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 19/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.147601:0:27439:0:(vvp_io.c:1339:vvp_io_write_start()) oosfile: write [1724534784, 1724535808) append 0 append lockpos 0 00000080:00200000:3.0:1787827173.147607:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421029 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.147623:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724534784, len 1024, copied 1024 00000080:00200000:3.0:1787827173.147627:0:27439:0:(rw26.c:933:ll_write_end()) page@00000000ef0e9245[3 000000005ce0a915 2 1 00000000a23cbd2a 1 0 421029] 00000080:00200000:3.0:1787827173.147630:0:27439:0:(rw26.c:933:ll_write_end()) vmpage @000000007d4118ff fffffc0001001 3:0 ffff9e34c3cb1828 421029 lru 00000080:00200000:3.0:1787827173.147635:0:27439:0:(rw26.c:933:ll_write_end()) osc-page@000000000f9bab11 210085: 1< 0 - - > 2< 860508160 0 0 0x0 0x0 | 000000004db7037e 0000000052729811 > 3< 0 0 > 4< 0 0 8 483938304 - | - - + - > 5< - - + - | 0 - | 383 - -> 00000080:00200000:3.0:1787827173.147637:0:27439:0:(rw26.c:933:ll_write_end()) end page@00000000ef0e9245 00000080:00200000:3.0:1787827173.147638:0:27439:0:(rw26.c:933:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1787827173.147640:0:27439:0:(vvp_io.c:1214:vvp_io_write_commit()) commit async pages: 1, from 0, to 1024 prio 0 00000008:00010000:3.0:1787827173.147648:0:27439:0:(osc_cache.c:565:osc_extent_merge()) ### extent: 000000001063fa34 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.147660:0:27439:0:(osc_cache.c:497:__osc_extent_remove()) ### extent: 000000001063fa34 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.147666:0:27439:0:(osc_cache.c:582:osc_extent_merge()) ### extent: 00000000df269729 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.147673:0:27439:0:(osc_cache.c:846:osc_extent_find()) ### extent: 00000000df269729 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.147683:0:27439:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff9e35d0e392f8, count 1, dirtied 1 00000080:00200000:3.0:1787827173.147687:0:27439:0:(vvp_io.c:1237:vvp_io_write_commit()) Committed 1 pages 1024 bytes, tot: 1024 00000080:00200000:3.0:1787827173.147689:0:27439:0:(vvp_io.c:1421:vvp_io_write_start()) oosfile: write bytes 0, result: 1024 00000020:00010000:3.0:1787827173.147695:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_release():245 00000020:00010000:3.0:1787827173.147698:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1787827173.147700:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c7d81d30 (ffff9e35da704cf8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1787827173.147704:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.147712:0:27439:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 17/0,0 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1787827173.147721:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_fini():72 00000020:00010000:3.0:1787827173.147724:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c7d81d30 (ffff9e35c614ce18/1) at cl_lock_fini():72 00000080:00200000:3.0:1787827173.147728:0:27439:0:(vvp_io.c:533:vvp_io_advance()) advancing 1024 bytes 00000080:00200000:3.0:1787827173.147731:0:27439:0:(file.c:2207:ll_file_io_generic()) Range unlock [421029, 421029] 00000080:00200000:3.0:1787827173.147735:0:27439:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787827173.147738:0:27439:0:(file.c:2240:ll_file_io_generic()) oosfile: 2 io complete with rc: 0, result: 1024, restart: 0 00000080:00200000:3.0:1787827173.147743:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724535808 i_blocks 3365288 00000080:00200000:3.0:1787827173.147746:0:27439:0:(file.c:2326:ll_file_io_generic()) iot: 2, result: 1024 00000080:00200008:3.0:1787827173.147753:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724535808, count: 1024 00000080:00200000:3.0:1787827173.147757:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421029 of 1024 to 1024 bytes 00000080:00200000:3.0:1787827173.147760:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724535808, len 1024, copied 1024 00000080:00200000:3.0:1787827173.147765:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724536832 i_blocks 3365288 00000080:00200000:3.0:1787827173.147768:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.147772:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724536832, count: 1024 00000080:00200000:3.0:1787827173.147775:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421029 of 2048 to 1024 bytes 00000080:00200000:3.0:1787827173.147777:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724536832, len 1024, copied 1024 00000080:00200000:3.0:1787827173.147781:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724537856 i_blocks 3365288 00000080:00200000:3.0:1787827173.147783:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.147786:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724537856, count: 1024 00000080:00200000:3.0:1787827173.147790:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421029 of 3072 to 1024 bytes 00000080:00200000:3.0:1787827173.147792:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724537856, len 1024, copied 1024 00000080:00200000:3.0:1787827173.147817:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724538880 i_blocks 3365288 00000080:00200000:3.0:1787827173.147821:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.147827:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724538880, count: 1024 00000080:00200000:3.0:1787827173.147830:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421030 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.147832:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 0, original count 1024 00000080:00200000:3.0:1787827173.147834:0:27439:0:(file.c:2090:ll_file_io_generic()) oosfile: write ppos: 1724538880, bytes: 1024 00000080:00200000:3.0:1787827173.147837:0:27439:0:(file.c:1848:ll_io_set_mirror()) oosfile: desiginated mirror: 0 00000020:00200000:3.0:1787827173.147840:0:27439:0:(cl_io.c:218:cl_io_rw_init()) header@000000007788600a[0x110, 368583, [0x200000402:0x27:0x0] hash] 00000020:00200000:3.0:1787827173.147841:0:27439:0:(cl_io.c:218:cl_io_rw_init()) io range: 2 [1724538880, 1724539904) 0 0 00000080:00200000:3.0:1787827173.147844:0:27439:0:(vvp_io.c:1889:vvp_io_init()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1787827173.147849:0:27439:0:(file.c:2170:ll_file_io_generic()) Range lock [421030, 421030] 00020000:00200000:3.0:1787827173.147852:0:27439:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 411 chunk: [1724538880, 1724539904) 1724539904, 1024 00020000:00200000:3.0:1787827173.147854:0:27439:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1787827173.147860:0:27439:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [421030, 421030] 00000008:00010000:3.0:1787827173.147865:0:27439:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1787827173.147868:0:27439:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff9e35c7d81d30, osc lock ffff9e35c2ddcc30, flags 0x0 00010000:00010000:3.0:1787827173.147875:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.147884:0:27439:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (860512256 860516351) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.147893:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 20/0,2 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.147901:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 20/0,2 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.147908:0:27439:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 19/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.147916:0:27439:0:(vvp_io.c:1339:vvp_io_write_start()) oosfile: write [1724538880, 1724539904) append 0 append lockpos 0 00000080:00200000:3.0:1787827173.147935:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421030 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.147953:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724538880, len 1024, copied 1024 00000080:00200000:3.0:1787827173.147957:0:27439:0:(rw26.c:933:ll_write_end()) page@0000000028ef2e88[3 000000005ce0a915 2 1 00000000a23cbd2a 1 0 421030] 00000080:00200000:3.0:1787827173.147960:0:27439:0:(rw26.c:933:ll_write_end()) vmpage @00000000cfed2465 fffffc0001001 3:0 ffff9e34c3cb1570 421030 lru 00000080:00200000:3.0:1787827173.147966:0:27439:0:(rw26.c:933:ll_write_end()) osc-page@00000000a83ee600 210086: 1< 0 - - > 2< 860512256 0 0 0x0 0x0 | 000000004db7037e 0000000052729811 > 3< 0 0 > 4< 0 0 8 483934208 - | - - + - > 5< - - + - | 0 - | 384 - -> 00000080:00200000:3.0:1787827173.147968:0:27439:0:(rw26.c:933:ll_write_end()) end page@0000000028ef2e88 00000080:00200000:3.0:1787827173.147969:0:27439:0:(rw26.c:933:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1787827173.147971:0:27439:0:(vvp_io.c:1214:vvp_io_write_commit()) commit async pages: 1, from 0, to 1024 prio 0 00000008:00010000:3.0:1787827173.147980:0:27439:0:(osc_cache.c:565:osc_extent_merge()) ### extent: 000000001063fa34 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.147993:0:27439:0:(osc_cache.c:497:__osc_extent_remove()) ### extent: 000000001063fa34 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.148001:0:27439:0:(osc_cache.c:582:osc_extent_merge()) ### extent: 00000000df269729 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.148010:0:27439:0:(osc_cache.c:846:osc_extent_find()) ### extent: 00000000df269729 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.148020:0:27439:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff9e35d0e392f8, count 1, dirtied 1 00000080:00200000:3.0:1787827173.148024:0:27439:0:(vvp_io.c:1237:vvp_io_write_commit()) Committed 1 pages 1024 bytes, tot: 1024 00000080:00200000:3.0:1787827173.148026:0:27439:0:(vvp_io.c:1421:vvp_io_write_start()) oosfile: write bytes 0, result: 1024 00000020:00010000:3.0:1787827173.148032:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_release():245 00000020:00010000:3.0:1787827173.148034:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1787827173.148037:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c7d81d30 (ffff9e35da704cf8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1787827173.148041:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.148050:0:27439:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 17/0,0 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1787827173.148058:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_fini():72 00000020:00010000:3.0:1787827173.148060:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c7d81d30 (ffff9e35c614ce18/1) at cl_lock_fini():72 00000080:00200000:3.0:1787827173.148063:0:27439:0:(vvp_io.c:533:vvp_io_advance()) advancing 1024 bytes 00000080:00200000:3.0:1787827173.148065:0:27439:0:(file.c:2207:ll_file_io_generic()) Range unlock [421030, 421030] 00000080:00200000:3.0:1787827173.148068:0:27439:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787827173.148070:0:27439:0:(file.c:2240:ll_file_io_generic()) oosfile: 2 io complete with rc: 0, result: 1024, restart: 0 00000080:00200000:3.0:1787827173.148073:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724539904 i_blocks 3365288 00000080:00200000:3.0:1787827173.148075:0:27439:0:(file.c:2326:ll_file_io_generic()) iot: 2, result: 1024 00000080:00200008:3.0:1787827173.148082:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724539904, count: 1024 00000080:00200000:3.0:1787827173.148087:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421030 of 1024 to 1024 bytes 00000080:00200000:3.0:1787827173.148090:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724539904, len 1024, copied 1024 00000080:00200000:3.0:1787827173.148096:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724540928 i_blocks 3365288 00000080:00200000:3.0:1787827173.148098:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.148102:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724540928, count: 1024 00000080:00200000:3.0:1787827173.148106:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421030 of 2048 to 1024 bytes 00000080:00200000:3.0:1787827173.148108:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724540928, len 1024, copied 1024 00000080:00200000:3.0:1787827173.148112:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724541952 i_blocks 3365288 00000080:00200000:3.0:1787827173.148114:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.148118:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724541952, count: 1024 00000080:00200000:3.0:1787827173.148121:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421030 of 3072 to 1024 bytes 00000080:00200000:3.0:1787827173.148123:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724541952, len 1024, copied 1024 00000080:00200000:3.0:1787827173.148127:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724542976 i_blocks 3365288 00000080:00200000:3.0:1787827173.148130:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.148134:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724542976, count: 1024 00000080:00200000:3.0:1787827173.148138:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421031 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.148140:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 0, original count 1024 00000080:00200000:3.0:1787827173.148142:0:27439:0:(file.c:2090:ll_file_io_generic()) oosfile: write ppos: 1724542976, bytes: 1024 00000080:00200000:3.0:1787827173.148144:0:27439:0:(file.c:1848:ll_io_set_mirror()) oosfile: desiginated mirror: 0 00000020:00200000:3.0:1787827173.148147:0:27439:0:(cl_io.c:218:cl_io_rw_init()) header@000000007788600a[0x110, 368584, [0x200000402:0x27:0x0] hash] 00000020:00200000:3.0:1787827173.148148:0:27439:0:(cl_io.c:218:cl_io_rw_init()) io range: 2 [1724542976, 1724544000) 0 0 00000080:00200000:3.0:1787827173.148150:0:27439:0:(vvp_io.c:1889:vvp_io_init()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1787827173.148154:0:27439:0:(file.c:2170:ll_file_io_generic()) Range lock [421031, 421031] 00020000:00200000:3.0:1787827173.148156:0:27439:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 411 chunk: [1724542976, 1724544000) 1724544000, 1024 00020000:00200000:3.0:1787827173.148158:0:27439:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1787827173.148164:0:27439:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [421031, 421031] 00000008:00010000:3.0:1787827173.148170:0:27439:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1787827173.148172:0:27439:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff9e35c7d81d30, osc lock ffff9e35c2ddcc30, flags 0x0 00010000:00010000:3.0:1787827173.148181:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.148189:0:27439:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (860516352 860520447) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.148198:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 20/0,2 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.148205:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 20/0,2 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.148212:0:27439:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 19/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.148221:0:27439:0:(vvp_io.c:1339:vvp_io_write_start()) oosfile: write [1724542976, 1724544000) append 0 append lockpos 0 00000080:00200000:3.0:1787827173.148226:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421031 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.148244:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724542976, len 1024, copied 1024 00000080:00200000:3.0:1787827173.148249:0:27439:0:(rw26.c:933:ll_write_end()) page@00000000df5500d9[3 000000005ce0a915 2 1 00000000a23cbd2a 1 0 421031] 00000080:00200000:3.0:1787827173.148252:0:27439:0:(rw26.c:933:ll_write_end()) vmpage @000000007ff9c4fa fffffc0001001 3:0 ffff9e34c3cb1e80 421031 lru 00000080:00200000:3.0:1787827173.148259:0:27439:0:(rw26.c:933:ll_write_end()) osc-page@00000000bc9b383d 210087: 1< 0 - - > 2< 860516352 0 0 0x0 0x0 | 000000004db7037e 0000000052729811 > 3< 0 0 > 4< 0 0 8 483930112 - | - - + - > 5< - - + - | 0 - | 385 - -> 00000080:00200000:3.0:1787827173.148261:0:27439:0:(rw26.c:933:ll_write_end()) end page@00000000df5500d9 00000080:00200000:3.0:1787827173.148262:0:27439:0:(rw26.c:933:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1787827173.148264:0:27439:0:(vvp_io.c:1214:vvp_io_write_commit()) commit async pages: 1, from 0, to 1024 prio 0 00000008:00010000:3.0:1787827173.148272:0:27439:0:(osc_cache.c:565:osc_extent_merge()) ### extent: 000000001063fa34 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.148287:0:27439:0:(osc_cache.c:497:__osc_extent_remove()) ### extent: 000000001063fa34 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.148295:0:27439:0:(osc_cache.c:582:osc_extent_merge()) ### extent: 00000000df269729 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.148303:0:27439:0:(osc_cache.c:846:osc_extent_find()) ### extent: 00000000df269729 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.148313:0:27439:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff9e35d0e392f8, count 1, dirtied 1 00000080:00200000:3.0:1787827173.148316:0:27439:0:(vvp_io.c:1237:vvp_io_write_commit()) Committed 1 pages 1024 bytes, tot: 1024 00000080:00200000:3.0:1787827173.148318:0:27439:0:(vvp_io.c:1421:vvp_io_write_start()) oosfile: write bytes 0, result: 1024 00000020:00010000:3.0:1787827173.148323:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_release():245 00000020:00010000:3.0:1787827173.148325:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1787827173.148328:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c7d81d30 (ffff9e35da704cf8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1787827173.148333:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.148342:0:27439:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 17/0,0 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1787827173.148351:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_fini():72 00000020:00010000:3.0:1787827173.148353:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c7d81d30 (ffff9e35c614ce18/1) at cl_lock_fini():72 00000080:00200000:3.0:1787827173.148356:0:27439:0:(vvp_io.c:533:vvp_io_advance()) advancing 1024 bytes 00000080:00200000:3.0:1787827173.148359:0:27439:0:(file.c:2207:ll_file_io_generic()) Range unlock [421031, 421031] 00000080:00200000:3.0:1787827173.148363:0:27439:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787827173.148366:0:27439:0:(file.c:2240:ll_file_io_generic()) oosfile: 2 io complete with rc: 0, result: 1024, restart: 0 00000080:00200000:3.0:1787827173.148371:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724544000 i_blocks 3365288 00000080:00200000:3.0:1787827173.148373:0:27439:0:(file.c:2326:ll_file_io_generic()) iot: 2, result: 1024 00000080:00200008:3.0:1787827173.148380:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724544000, count: 1024 00000080:00200000:3.0:1787827173.148384:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421031 of 1024 to 1024 bytes 00000080:00200000:3.0:1787827173.148388:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724544000, len 1024, copied 1024 00000080:00200000:3.0:1787827173.148393:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724545024 i_blocks 3365288 00000080:00200000:3.0:1787827173.148396:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.148400:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724545024, count: 1024 00000080:00200000:3.0:1787827173.148403:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421031 of 2048 to 1024 bytes 00000080:00200000:3.0:1787827173.148405:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724545024, len 1024, copied 1024 00000080:00200000:3.0:1787827173.148409:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724546048 i_blocks 3365288 00000080:00200000:3.0:1787827173.148411:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.148415:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724546048, count: 1024 00000080:00200000:3.0:1787827173.148419:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421031 of 3072 to 1024 bytes 00000080:00200000:3.0:1787827173.148421:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724546048, len 1024, copied 1024 00000080:00200000:3.0:1787827173.148424:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724547072 i_blocks 3365288 00000080:00200000:3.0:1787827173.148425:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.148428:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724547072, count: 1024 00000080:00200000:3.0:1787827173.148430:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421032 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.148431:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 0, original count 1024 00000080:00200000:3.0:1787827173.148432:0:27439:0:(file.c:2090:ll_file_io_generic()) oosfile: write ppos: 1724547072, bytes: 1024 00000080:00200000:3.0:1787827173.148434:0:27439:0:(file.c:1848:ll_io_set_mirror()) oosfile: desiginated mirror: 0 00000020:00200000:3.0:1787827173.148436:0:27439:0:(cl_io.c:218:cl_io_rw_init()) header@000000007788600a[0x110, 368585, [0x200000402:0x27:0x0] hash] 00000020:00200000:3.0:1787827173.148438:0:27439:0:(cl_io.c:218:cl_io_rw_init()) io range: 2 [1724547072, 1724548096) 0 0 00000080:00200000:3.0:1787827173.148439:0:27439:0:(vvp_io.c:1889:vvp_io_init()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1787827173.148445:0:27439:0:(file.c:2170:ll_file_io_generic()) Range lock [421032, 421032] 00020000:00200000:3.0:1787827173.148448:0:27439:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 411 chunk: [1724547072, 1724548096) 1724548096, 1024 00020000:00200000:3.0:1787827173.148450:0:27439:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1787827173.148456:0:27439:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [421032, 421032] 00000008:00010000:3.0:1787827173.148461:0:27439:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1787827173.148464:0:27439:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff9e35c7d81d30, osc lock ffff9e35c2ddcc30, flags 0x0 00010000:00010000:3.0:1787827173.148472:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.148481:0:27439:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (860520448 860524543) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.148490:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 20/0,2 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.148496:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 20/0,2 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.148500:0:27439:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 19/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.148505:0:27439:0:(vvp_io.c:1339:vvp_io_write_start()) oosfile: write [1724547072, 1724548096) append 0 append lockpos 0 00000080:00200000:3.0:1787827173.148508:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421032 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.148521:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724547072, len 1024, copied 1024 00000080:00200000:3.0:1787827173.148524:0:27439:0:(rw26.c:933:ll_write_end()) page@000000005c782cb8[3 000000005ce0a915 2 1 00000000a23cbd2a 1 0 421032] 00000080:00200000:3.0:1787827173.148526:0:27439:0:(rw26.c:933:ll_write_end()) vmpage @000000003432d23f fffffc0001001 3:0 ffff9e34c3cb1740 421032 lru 00000080:00200000:3.0:1787827173.148529:0:27439:0:(rw26.c:933:ll_write_end()) osc-page@0000000094476b6b 210088: 1< 0 - - > 2< 860520448 0 0 0x0 0x0 | 000000004db7037e 0000000052729811 > 3< 0 0 > 4< 0 0 8 483926016 - | - - + - > 5< - - + - | 0 - | 386 - -> 00000080:00200000:3.0:1787827173.148531:0:27439:0:(rw26.c:933:ll_write_end()) end page@000000005c782cb8 00000080:00200000:3.0:1787827173.148531:0:27439:0:(rw26.c:933:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1787827173.148533:0:27439:0:(vvp_io.c:1214:vvp_io_write_commit()) commit async pages: 1, from 0, to 1024 prio 0 00000008:00010000:3.0:1787827173.148542:0:27439:0:(osc_cache.c:565:osc_extent_merge()) ### extent: 000000001063fa34 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.148554:0:27439:0:(osc_cache.c:497:__osc_extent_remove()) ### extent: 000000001063fa34 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.148561:0:27439:0:(osc_cache.c:582:osc_extent_merge()) ### extent: 00000000df269729 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.148703:0:27439:0:(osc_cache.c:846:osc_extent_find()) ### extent: 00000000df269729 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.148716:0:27439:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff9e35d0e392f8, count 1, dirtied 1 00000080:00200000:3.0:1787827173.148720:0:27439:0:(vvp_io.c:1237:vvp_io_write_commit()) Committed 1 pages 1024 bytes, tot: 1024 00000080:00200000:3.0:1787827173.148722:0:27439:0:(vvp_io.c:1421:vvp_io_write_start()) oosfile: write bytes 0, result: 1024 00000020:00010000:3.0:1787827173.148727:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_release():245 00000020:00010000:3.0:1787827173.148730:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1787827173.148733:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c7d81d30 (ffff9e35da704cf8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1787827173.148776:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.148784:0:27439:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 17/0,0 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1787827173.148794:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_fini():72 00000020:00010000:3.0:1787827173.148814:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c7d81d30 (ffff9e35c614ce18/1) at cl_lock_fini():72 00000080:00200000:3.0:1787827173.148820:0:27439:0:(vvp_io.c:533:vvp_io_advance()) advancing 1024 bytes 00000080:00200000:3.0:1787827173.148823:0:27439:0:(file.c:2207:ll_file_io_generic()) Range unlock [421032, 421032] 00000080:00200000:3.0:1787827173.148827:0:27439:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787827173.148830:0:27439:0:(file.c:2240:ll_file_io_generic()) oosfile: 2 io complete with rc: 0, result: 1024, restart: 0 00000080:00200000:3.0:1787827173.148835:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724548096 i_blocks 3365288 00000080:00200000:3.0:1787827173.148838:0:27439:0:(file.c:2326:ll_file_io_generic()) iot: 2, result: 1024 00000080:00200008:3.0:1787827173.148846:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724548096, count: 1024 00000080:00200000:3.0:1787827173.148850:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421032 of 1024 to 1024 bytes 00000080:00200000:3.0:1787827173.148854:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724548096, len 1024, copied 1024 00000080:00200000:3.0:1787827173.148859:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724549120 i_blocks 3365288 00000080:00200000:3.0:1787827173.148861:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.148866:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724549120, count: 1024 00000080:00200000:3.0:1787827173.148869:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421032 of 2048 to 1024 bytes 00000080:00200000:3.0:1787827173.148871:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724549120, len 1024, copied 1024 00000080:00200000:3.0:1787827173.148876:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724550144 i_blocks 3365288 00000080:00200000:3.0:1787827173.148878:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.148882:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724550144, count: 1024 00000080:00200000:3.0:1787827173.148885:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421032 of 3072 to 1024 bytes 00000080:00200000:3.0:1787827173.148887:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724550144, len 1024, copied 1024 00000080:00200000:3.0:1787827173.148891:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724551168 i_blocks 3365288 00000080:00200000:3.0:1787827173.148894:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.148898:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724551168, count: 1024 00000080:00200000:3.0:1787827173.148902:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421033 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.148904:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 0, original count 1024 00000080:00200000:3.0:1787827173.148906:0:27439:0:(file.c:2090:ll_file_io_generic()) oosfile: write ppos: 1724551168, bytes: 1024 00000080:00200000:3.0:1787827173.148909:0:27439:0:(file.c:1848:ll_io_set_mirror()) oosfile: desiginated mirror: 0 00000020:00200000:3.0:1787827173.148913:0:27439:0:(cl_io.c:218:cl_io_rw_init()) header@000000007788600a[0x110, 368586, [0x200000402:0x27:0x0] hash] 00000020:00200000:3.0:1787827173.148915:0:27439:0:(cl_io.c:218:cl_io_rw_init()) io range: 2 [1724551168, 1724552192) 0 0 00000080:00200000:3.0:1787827173.148931:0:27439:0:(vvp_io.c:1889:vvp_io_init()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1787827173.148937:0:27439:0:(file.c:2170:ll_file_io_generic()) Range lock [421033, 421033] 00020000:00200000:3.0:1787827173.148940:0:27439:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 411 chunk: [1724551168, 1724552192) 1724552192, 1024 00020000:00200000:3.0:1787827173.148943:0:27439:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1787827173.148949:0:27439:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [421033, 421033] 00000008:00010000:3.0:1787827173.148957:0:27439:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1787827173.148959:0:27439:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff9e35c7d81d30, osc lock ffff9e35c2ddcc30, flags 0x0 00010000:00010000:3.0:1787827173.148965:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.148972:0:27439:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (860524544 860528639) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.148977:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 20/0,2 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.148982:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 20/0,2 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.148989:0:27439:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 19/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.148998:0:27439:0:(vvp_io.c:1339:vvp_io_write_start()) oosfile: write [1724551168, 1724552192) append 0 append lockpos 0 00000080:00200000:3.0:1787827173.149003:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421033 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.149031:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724551168, len 1024, copied 1024 00000080:00200000:3.0:1787827173.149036:0:27439:0:(rw26.c:933:ll_write_end()) page@00000000aa3fe703[3 000000005ce0a915 2 1 00000000a23cbd2a 1 0 421033] 00000080:00200000:3.0:1787827173.149039:0:27439:0:(rw26.c:933:ll_write_end()) vmpage @00000000682d1fc6 fffffc0001001 3:0 ffff9e34c3cb2658 421033 lru 00000080:00200000:3.0:1787827173.149045:0:27439:0:(rw26.c:933:ll_write_end()) osc-page@0000000023161f2a 210089: 1< 0 - - > 2< 860524544 0 0 0x0 0x0 | 000000004db7037e 0000000052729811 > 3< 0 0 > 4< 0 0 8 483921920 - | - - + - > 5< - - + - | 0 - | 387 - -> 00000080:00200000:3.0:1787827173.149048:0:27439:0:(rw26.c:933:ll_write_end()) end page@00000000aa3fe703 00000080:00200000:3.0:1787827173.149049:0:27439:0:(rw26.c:933:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1787827173.149051:0:27439:0:(vvp_io.c:1214:vvp_io_write_commit()) commit async pages: 1, from 0, to 1024 prio 0 00000008:00010000:3.0:1787827173.149060:0:27439:0:(osc_cache.c:565:osc_extent_merge()) ### extent: 000000001063fa34 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.149075:0:27439:0:(osc_cache.c:497:__osc_extent_remove()) ### extent: 000000001063fa34 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.149084:0:27439:0:(osc_cache.c:582:osc_extent_merge()) ### extent: 00000000df269729 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.149092:0:27439:0:(osc_cache.c:846:osc_extent_find()) ### extent: 00000000df269729 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.149104:0:27439:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff9e35d0e392f8, count 1, dirtied 1 00000080:00200000:3.0:1787827173.149107:0:27439:0:(vvp_io.c:1237:vvp_io_write_commit()) Committed 1 pages 1024 bytes, tot: 1024 00000080:00200000:3.0:1787827173.149110:0:27439:0:(vvp_io.c:1421:vvp_io_write_start()) oosfile: write bytes 0, result: 1024 00000020:00010000:3.0:1787827173.149116:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_release():245 00000020:00010000:3.0:1787827173.149119:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1787827173.149121:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c7d81d30 (ffff9e35da704cf8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1787827173.149126:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.149133:0:27439:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 17/0,0 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1787827173.149139:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_fini():72 00000020:00010000:3.0:1787827173.149141:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c7d81d30 (ffff9e35c614ce18/1) at cl_lock_fini():72 00000080:00200000:3.0:1787827173.149144:0:27439:0:(vvp_io.c:533:vvp_io_advance()) advancing 1024 bytes 00000080:00200000:3.0:1787827173.149146:0:27439:0:(file.c:2207:ll_file_io_generic()) Range unlock [421033, 421033] 00000080:00200000:3.0:1787827173.149149:0:27439:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787827173.149151:0:27439:0:(file.c:2240:ll_file_io_generic()) oosfile: 2 io complete with rc: 0, result: 1024, restart: 0 00000080:00200000:3.0:1787827173.149155:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724552192 i_blocks 3365288 00000080:00200000:3.0:1787827173.149156:0:27439:0:(file.c:2326:ll_file_io_generic()) iot: 2, result: 1024 00000080:00200008:3.0:1787827173.149162:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724552192, count: 1024 00000080:00200000:3.0:1787827173.149166:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421033 of 1024 to 1024 bytes 00000080:00200000:3.0:1787827173.149168:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724552192, len 1024, copied 1024 00000080:00200000:3.0:1787827173.149171:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724553216 i_blocks 3365288 00000080:00200000:3.0:1787827173.149172:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.149175:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724553216, count: 1024 00000080:00200000:3.0:1787827173.149177:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421033 of 2048 to 1024 bytes 00000080:00200000:3.0:1787827173.149178:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724553216, len 1024, copied 1024 00000080:00200000:3.0:1787827173.149180:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724554240 i_blocks 3365288 00000080:00200000:3.0:1787827173.149182:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.149184:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724554240, count: 1024 00000080:00200000:3.0:1787827173.149186:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421033 of 3072 to 1024 bytes 00000080:00200000:3.0:1787827173.149187:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724554240, len 1024, copied 1024 00000080:00200000:3.0:1787827173.149189:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724555264 i_blocks 3365288 00000080:00200000:3.0:1787827173.149191:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.149193:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724555264, count: 1024 00000080:00200000:3.0:1787827173.149195:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421034 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.149196:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 0, original count 1024 00000080:00200000:3.0:1787827173.149197:0:27439:0:(file.c:2090:ll_file_io_generic()) oosfile: write ppos: 1724555264, bytes: 1024 00000080:00200000:3.0:1787827173.149199:0:27439:0:(file.c:1848:ll_io_set_mirror()) oosfile: desiginated mirror: 0 00000020:00200000:3.0:1787827173.149201:0:27439:0:(cl_io.c:218:cl_io_rw_init()) header@000000007788600a[0x110, 368587, [0x200000402:0x27:0x0] hash] 00000020:00200000:3.0:1787827173.149202:0:27439:0:(cl_io.c:218:cl_io_rw_init()) io range: 2 [1724555264, 1724556288) 0 0 00000080:00200000:3.0:1787827173.149203:0:27439:0:(vvp_io.c:1889:vvp_io_init()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1787827173.149207:0:27439:0:(file.c:2170:ll_file_io_generic()) Range lock [421034, 421034] 00020000:00200000:3.0:1787827173.149209:0:27439:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 411 chunk: [1724555264, 1724556288) 1724556288, 1024 00020000:00200000:3.0:1787827173.149210:0:27439:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1787827173.149214:0:27439:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [421034, 421034] 00000008:00010000:3.0:1787827173.149218:0:27439:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1787827173.149220:0:27439:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff9e35c7d81d30, osc lock ffff9e35c2ddcc30, flags 0x0 00010000:00010000:3.0:1787827173.149226:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.149235:0:27439:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (860528640 860532735) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.149256:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 20/0,2 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.149264:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 20/0,2 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.149271:0:27439:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 19/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.149279:0:27439:0:(vvp_io.c:1339:vvp_io_write_start()) oosfile: write [1724555264, 1724556288) append 0 append lockpos 0 00000080:00200000:3.0:1787827173.149285:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421034 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.149299:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724555264, len 1024, copied 1024 00000080:00200000:3.0:1787827173.149302:0:27439:0:(rw26.c:933:ll_write_end()) page@000000006e0ea0c0[3 000000005ce0a915 2 1 00000000a23cbd2a 1 0 421034] 00000080:00200000:3.0:1787827173.149304:0:27439:0:(rw26.c:933:ll_write_end()) vmpage @000000001afb6ab2 fffffc0001001 3:0 ffff9e34c3cb23a0 421034 lru 00000080:00200000:3.0:1787827173.149308:0:27439:0:(rw26.c:933:ll_write_end()) osc-page@00000000b8a01e80 210090: 1< 0 - - > 2< 860528640 0 0 0x0 0x0 | 000000004db7037e 0000000052729811 > 3< 0 0 > 4< 0 0 8 483917824 - | - - + - > 5< - - + - | 0 - | 388 - -> 00000080:00200000:3.0:1787827173.149310:0:27439:0:(rw26.c:933:ll_write_end()) end page@000000006e0ea0c0 00000080:00200000:3.0:1787827173.149311:0:27439:0:(rw26.c:933:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1787827173.149312:0:27439:0:(vvp_io.c:1214:vvp_io_write_commit()) commit async pages: 1, from 0, to 1024 prio 0 00000008:00010000:3.0:1787827173.149321:0:27439:0:(osc_cache.c:565:osc_extent_merge()) ### extent: 000000001063fa34 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.149332:0:27439:0:(osc_cache.c:497:__osc_extent_remove()) ### extent: 000000001063fa34 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.149339:0:27439:0:(osc_cache.c:582:osc_extent_merge()) ### extent: 00000000df269729 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.149346:0:27439:0:(osc_cache.c:846:osc_extent_find()) ### extent: 00000000df269729 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.149357:0:27439:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff9e35d0e392f8, count 1, dirtied 1 00000080:00200000:3.0:1787827173.149361:0:27439:0:(vvp_io.c:1237:vvp_io_write_commit()) Committed 1 pages 1024 bytes, tot: 1024 00000080:00200000:3.0:1787827173.149363:0:27439:0:(vvp_io.c:1421:vvp_io_write_start()) oosfile: write bytes 0, result: 1024 00000020:00010000:3.0:1787827173.149369:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_release():245 00000020:00010000:3.0:1787827173.149372:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1787827173.149375:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c7d81d30 (ffff9e35da704cf8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1787827173.149380:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.149389:0:27439:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 17/0,0 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1787827173.149399:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_fini():72 00000020:00010000:3.0:1787827173.149401:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c7d81d30 (ffff9e35c614ce18/1) at cl_lock_fini():72 00000080:00200000:3.0:1787827173.149405:0:27439:0:(vvp_io.c:533:vvp_io_advance()) advancing 1024 bytes 00000080:00200000:3.0:1787827173.149407:0:27439:0:(file.c:2207:ll_file_io_generic()) Range unlock [421034, 421034] 00000080:00200000:3.0:1787827173.149411:0:27439:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787827173.149413:0:27439:0:(file.c:2240:ll_file_io_generic()) oosfile: 2 io complete with rc: 0, result: 1024, restart: 0 00000080:00200000:3.0:1787827173.149417:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724556288 i_blocks 3365288 00000080:00200000:3.0:1787827173.149419:0:27439:0:(file.c:2326:ll_file_io_generic()) iot: 2, result: 1024 00000080:00200008:3.0:1787827173.149425:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724556288, count: 1024 00000080:00200000:3.0:1787827173.149429:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421034 of 1024 to 1024 bytes 00000080:00200000:3.0:1787827173.149433:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724556288, len 1024, copied 1024 00000080:00200000:3.0:1787827173.149438:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724557312 i_blocks 3365288 00000080:00200000:3.0:1787827173.149441:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.149445:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724557312, count: 1024 00000080:00200000:3.0:1787827173.149449:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421034 of 2048 to 1024 bytes 00000080:00200000:3.0:1787827173.149452:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724557312, len 1024, copied 1024 00000080:00200000:3.0:1787827173.149457:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724558336 i_blocks 3365288 00000080:00200000:3.0:1787827173.149460:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.149464:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724558336, count: 1024 00000080:00200000:3.0:1787827173.149468:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421034 of 3072 to 1024 bytes 00000080:00200000:3.0:1787827173.149471:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724558336, len 1024, copied 1024 00000080:00200000:3.0:1787827173.149476:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724559360 i_blocks 3365288 00000080:00200000:3.0:1787827173.149479:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.149484:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724559360, count: 1024 00000080:00200000:3.0:1787827173.149487:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421035 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.149489:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 0, original count 1024 00000080:00200000:3.0:1787827173.149490:0:27439:0:(file.c:2090:ll_file_io_generic()) oosfile: write ppos: 1724559360, bytes: 1024 00000080:00200000:3.0:1787827173.149492:0:27439:0:(file.c:1848:ll_io_set_mirror()) oosfile: desiginated mirror: 0 00000020:00200000:3.0:1787827173.149495:0:27439:0:(cl_io.c:218:cl_io_rw_init()) header@000000007788600a[0x110, 368588, [0x200000402:0x27:0x0] hash] 00000020:00200000:3.0:1787827173.149497:0:27439:0:(cl_io.c:218:cl_io_rw_init()) io range: 2 [1724559360, 1724560384) 0 0 00000080:00200000:3.0:1787827173.149499:0:27439:0:(vvp_io.c:1889:vvp_io_init()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1787827173.149504:0:27439:0:(file.c:2170:ll_file_io_generic()) Range lock [421035, 421035] 00020000:00200000:3.0:1787827173.149506:0:27439:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 411 chunk: [1724559360, 1724560384) 1724560384, 1024 00020000:00200000:3.0:1787827173.149508:0:27439:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1787827173.149513:0:27439:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [421035, 421035] 00000008:00010000:3.0:1787827173.149519:0:27439:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1787827173.149521:0:27439:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff9e35c7d81d30, osc lock ffff9e35c2ddcc30, flags 0x0 00010000:00010000:3.0:1787827173.149531:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.149541:0:27439:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (860532736 860536831) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.149550:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 20/0,2 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.149559:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 20/0,2 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.149567:0:27439:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 19/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.149575:0:27439:0:(vvp_io.c:1339:vvp_io_write_start()) oosfile: write [1724559360, 1724560384) append 0 append lockpos 0 00000080:00200000:3.0:1787827173.149582:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421035 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.149601:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724559360, len 1024, copied 1024 00000080:00200000:3.0:1787827173.149606:0:27439:0:(rw26.c:933:ll_write_end()) page@00000000a0f6d256[3 000000005ce0a915 2 1 00000000a23cbd2a 1 0 421035] 00000080:00200000:3.0:1787827173.149610:0:27439:0:(rw26.c:933:ll_write_end()) vmpage @0000000083ecfd12 fffffc0001001 3:0 ffff9e34c3cb2d98 421035 lru 00000080:00200000:3.0:1787827173.149616:0:27439:0:(rw26.c:933:ll_write_end()) osc-page@00000000a2a2c58f 210091: 1< 0 - - > 2< 860532736 0 0 0x0 0x0 | 000000004db7037e 0000000052729811 > 3< 0 0 > 4< 0 0 8 483913728 - | - - + - > 5< - - + - | 0 - | 389 - -> 00000080:00200000:3.0:1787827173.149618:0:27439:0:(rw26.c:933:ll_write_end()) end page@00000000a0f6d256 00000080:00200000:3.0:1787827173.149620:0:27439:0:(rw26.c:933:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1787827173.149623:0:27439:0:(vvp_io.c:1214:vvp_io_write_commit()) commit async pages: 1, from 0, to 1024 prio 0 00000008:00010000:3.0:1787827173.149632:0:27439:0:(osc_cache.c:565:osc_extent_merge()) ### extent: 000000001063fa34 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.149647:0:27439:0:(osc_cache.c:497:__osc_extent_remove()) ### extent: 000000001063fa34 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.149654:0:27439:0:(osc_cache.c:582:osc_extent_merge()) ### extent: 00000000df269729 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.149662:0:27439:0:(osc_cache.c:846:osc_extent_find()) ### extent: 00000000df269729 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.149677:0:27439:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff9e35d0e392f8, count 1, dirtied 1 00000080:00200000:3.0:1787827173.149681:0:27439:0:(vvp_io.c:1237:vvp_io_write_commit()) Committed 1 pages 1024 bytes, tot: 1024 00000080:00200000:3.0:1787827173.149683:0:27439:0:(vvp_io.c:1421:vvp_io_write_start()) oosfile: write bytes 0, result: 1024 00000020:00010000:3.0:1787827173.149691:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_release():245 00000020:00010000:3.0:1787827173.149694:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1787827173.149697:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c7d81d30 (ffff9e35da704cf8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1787827173.149701:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.149710:0:27439:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 17/0,0 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1787827173.149720:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_fini():72 00000020:00010000:3.0:1787827173.149723:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c7d81d30 (ffff9e35c614ce18/1) at cl_lock_fini():72 00000080:00200000:3.0:1787827173.149729:0:27439:0:(vvp_io.c:533:vvp_io_advance()) advancing 1024 bytes 00000080:00200000:3.0:1787827173.149733:0:27439:0:(file.c:2207:ll_file_io_generic()) Range unlock [421035, 421035] 00000080:00200000:3.0:1787827173.149738:0:27439:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787827173.149741:0:27439:0:(file.c:2240:ll_file_io_generic()) oosfile: 2 io complete with rc: 0, result: 1024, restart: 0 00000080:00200000:3.0:1787827173.149747:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724560384 i_blocks 3365288 00000080:00200000:3.0:1787827173.149749:0:27439:0:(file.c:2326:ll_file_io_generic()) iot: 2, result: 1024 00000080:00200008:3.0:1787827173.149757:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724560384, count: 1024 00000080:00200000:3.0:1787827173.149762:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421035 of 1024 to 1024 bytes 00000080:00200000:3.0:1787827173.149766:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724560384, len 1024, copied 1024 00000080:00200000:3.0:1787827173.149772:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724561408 i_blocks 3365288 00000080:00200000:3.0:1787827173.149775:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.149778:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724561408, count: 1024 00000080:00200000:3.0:1787827173.149782:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421035 of 2048 to 1024 bytes 00000080:00200000:3.0:1787827173.149785:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724561408, len 1024, copied 1024 00000080:00200000:3.0:1787827173.149790:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724562432 i_blocks 3365288 00000080:00200000:3.0:1787827173.149792:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.149817:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724562432, count: 1024 00000080:00200000:3.0:1787827173.149822:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421035 of 3072 to 1024 bytes 00000080:00200000:3.0:1787827173.149825:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724562432, len 1024, copied 1024 00000080:00200000:3.0:1787827173.149830:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724563456 i_blocks 3365288 00000080:00200000:3.0:1787827173.149833:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.149838:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724563456, count: 1024 00000080:00200000:3.0:1787827173.149841:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421036 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.149843:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 0, original count 1024 00000080:00200000:3.0:1787827173.149846:0:27439:0:(file.c:2090:ll_file_io_generic()) oosfile: write ppos: 1724563456, bytes: 1024 00000080:00200000:3.0:1787827173.149848:0:27439:0:(file.c:1848:ll_io_set_mirror()) oosfile: desiginated mirror: 0 00000020:00200000:3.0:1787827173.149852:0:27439:0:(cl_io.c:218:cl_io_rw_init()) header@000000007788600a[0x110, 368589, [0x200000402:0x27:0x0] hash] 00000020:00200000:3.0:1787827173.149854:0:27439:0:(cl_io.c:218:cl_io_rw_init()) io range: 2 [1724563456, 1724564480) 0 0 00000080:00200000:3.0:1787827173.149856:0:27439:0:(vvp_io.c:1889:vvp_io_init()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1787827173.149861:0:27439:0:(file.c:2170:ll_file_io_generic()) Range lock [421036, 421036] 00020000:00200000:3.0:1787827173.149864:0:27439:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 411 chunk: [1724563456, 1724564480) 1724564480, 1024 00020000:00200000:3.0:1787827173.149867:0:27439:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1787827173.149872:0:27439:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [421036, 421036] 00000008:00010000:3.0:1787827173.149878:0:27439:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1787827173.149880:0:27439:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff9e35c7d81d30, osc lock ffff9e35c2ddcc30, flags 0x0 00010000:00010000:3.0:1787827173.149888:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.149896:0:27439:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (860536832 860540927) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.149904:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 20/0,2 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.149911:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 20/0,2 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.149930:0:27439:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 19/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.149939:0:27439:0:(vvp_io.c:1339:vvp_io_write_start()) oosfile: write [1724563456, 1724564480) append 0 append lockpos 0 00000080:00200000:3.0:1787827173.149944:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421036 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.149962:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724563456, len 1024, copied 1024 00000080:00200000:3.0:1787827173.149966:0:27439:0:(rw26.c:933:ll_write_end()) page@00000000870b63d9[3 000000005ce0a915 2 1 00000000a23cbd2a 1 0 421036] 00000080:00200000:3.0:1787827173.149970:0:27439:0:(rw26.c:933:ll_write_end()) vmpage @000000002fe6900b fffffc0001001 3:0 ffff9e34c3cb2000 421036 lru 00000080:00200000:3.0:1787827173.149976:0:27439:0:(rw26.c:933:ll_write_end()) osc-page@00000000822d4ad2 210092: 1< 0 - - > 2< 860536832 0 0 0x0 0x0 | 000000004db7037e 0000000052729811 > 3< 0 0 > 4< 0 0 8 483909632 - | - - + - > 5< - - + - | 0 - | 390 - -> 00000080:00200000:3.0:1787827173.149978:0:27439:0:(rw26.c:933:ll_write_end()) end page@00000000870b63d9 00000080:00200000:3.0:1787827173.149981:0:27439:0:(rw26.c:933:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1787827173.149983:0:27439:0:(vvp_io.c:1214:vvp_io_write_commit()) commit async pages: 1, from 0, to 1024 prio 0 00000008:00010000:3.0:1787827173.149992:0:27439:0:(osc_cache.c:565:osc_extent_merge()) ### extent: 000000001063fa34 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.150008:0:27439:0:(osc_cache.c:497:__osc_extent_remove()) ### extent: 000000001063fa34 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.150016:0:27439:0:(osc_cache.c:582:osc_extent_merge()) ### extent: 00000000df269729 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.150024:0:27439:0:(osc_cache.c:846:osc_extent_find()) ### extent: 00000000df269729 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.150038:0:27439:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff9e35d0e392f8, count 1, dirtied 1 00000080:00200000:3.0:1787827173.150042:0:27439:0:(vvp_io.c:1237:vvp_io_write_commit()) Committed 1 pages 1024 bytes, tot: 1024 00000080:00200000:3.0:1787827173.150045:0:27439:0:(vvp_io.c:1421:vvp_io_write_start()) oosfile: write bytes 0, result: 1024 00000020:00010000:3.0:1787827173.150053:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_release():245 00000020:00010000:3.0:1787827173.150056:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1787827173.150059:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c7d81d30 (ffff9e35da704cf8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1787827173.150064:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.150072:0:27439:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 17/0,0 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1787827173.150080:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_fini():72 00000020:00010000:3.0:1787827173.150083:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c7d81d30 (ffff9e35c614ce18/1) at cl_lock_fini():72 00000080:00200000:3.0:1787827173.150087:0:27439:0:(vvp_io.c:533:vvp_io_advance()) advancing 1024 bytes 00000080:00200000:3.0:1787827173.150091:0:27439:0:(file.c:2207:ll_file_io_generic()) Range unlock [421036, 421036] 00000080:00200000:3.0:1787827173.150097:0:27439:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787827173.150105:0:27439:0:(file.c:2240:ll_file_io_generic()) oosfile: 2 io complete with rc: 0, result: 1024, restart: 0 00000080:00200000:3.0:1787827173.150111:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724564480 i_blocks 3365288 00000080:00200000:3.0:1787827173.150116:0:27439:0:(file.c:2326:ll_file_io_generic()) iot: 2, result: 1024 00000080:00200008:3.0:1787827173.150124:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724564480, count: 1024 00000080:00200000:3.0:1787827173.150129:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421036 of 1024 to 1024 bytes 00000080:00200000:3.0:1787827173.150134:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724564480, len 1024, copied 1024 00000080:00200000:3.0:1787827173.150144:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724565504 i_blocks 3365288 00000080:00200000:3.0:1787827173.150151:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.150158:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724565504, count: 1024 00000080:00200000:3.0:1787827173.150167:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421036 of 2048 to 1024 bytes 00000080:00200000:3.0:1787827173.150174:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724565504, len 1024, copied 1024 00000080:00200000:3.0:1787827173.150181:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724566528 i_blocks 3365288 00000080:00200000:3.0:1787827173.150186:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.150190:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724566528, count: 1024 00000080:00200000:3.0:1787827173.150195:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421036 of 3072 to 1024 bytes 00000080:00200000:3.0:1787827173.150200:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724566528, len 1024, copied 1024 00000080:00200000:3.0:1787827173.150208:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724567552 i_blocks 3365288 00000080:00200000:3.0:1787827173.150212:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.150217:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724567552, count: 1024 00000080:00200000:3.0:1787827173.150224:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421037 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.150228:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 0, original count 1024 00000080:00200000:3.0:1787827173.150234:0:27439:0:(file.c:2090:ll_file_io_generic()) oosfile: write ppos: 1724567552, bytes: 1024 00000080:00200000:3.0:1787827173.150237:0:27439:0:(file.c:1848:ll_io_set_mirror()) oosfile: desiginated mirror: 0 00000020:00200000:3.0:1787827173.150244:0:27439:0:(cl_io.c:218:cl_io_rw_init()) header@000000007788600a[0x110, 368590, [0x200000402:0x27:0x0] hash] 00000020:00200000:3.0:1787827173.150247:0:27439:0:(cl_io.c:218:cl_io_rw_init()) io range: 2 [1724567552, 1724568576) 0 0 00000080:00200000:3.0:1787827173.150254:0:27439:0:(vvp_io.c:1889:vvp_io_init()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1787827173.150264:0:27439:0:(file.c:2170:ll_file_io_generic()) Range lock [421037, 421037] 00020000:00200000:3.0:1787827173.150269:0:27439:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 411 chunk: [1724567552, 1724568576) 1724568576, 1024 00020000:00200000:3.0:1787827173.150275:0:27439:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1787827173.150283:0:27439:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [421037, 421037] 00000008:00010000:3.0:1787827173.150289:0:27439:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1787827173.150292:0:27439:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff9e35c7d81d30, osc lock ffff9e35c2ddcc30, flags 0x0 00010000:00010000:3.0:1787827173.150300:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.150308:0:27439:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (860540928 860545023) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.150319:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 20/0,2 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.150329:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 20/0,2 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.150336:0:27439:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 19/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.150347:0:27439:0:(vvp_io.c:1339:vvp_io_write_start()) oosfile: write [1724567552, 1724568576) append 0 append lockpos 0 00000080:00200000:3.0:1787827173.150355:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421037 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.150375:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724567552, len 1024, copied 1024 00000080:00200000:3.0:1787827173.150383:0:27439:0:(rw26.c:933:ll_write_end()) page@00000000bb23fd38[3 000000005ce0a915 2 1 00000000a23cbd2a 1 0 421037] 00000080:00200000:3.0:1787827173.150390:0:27439:0:(rw26.c:933:ll_write_end()) vmpage @00000000d878eb32 fffffc0001001 3:0 ffff9e34c3cb22b8 421037 lru 00000080:00200000:3.0:1787827173.150399:0:27439:0:(rw26.c:933:ll_write_end()) osc-page@000000007f926b01 210093: 1< 0 - - > 2< 860540928 0 0 0x0 0x0 | 000000004db7037e 0000000052729811 > 3< 0 0 > 4< 0 0 8 483905536 - | - - + - > 5< - - + - | 0 - | 391 - -> 00000080:00200000:3.0:1787827173.150403:0:27439:0:(rw26.c:933:ll_write_end()) end page@00000000bb23fd38 00000080:00200000:3.0:1787827173.150407:0:27439:0:(rw26.c:933:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1787827173.150411:0:27439:0:(vvp_io.c:1214:vvp_io_write_commit()) commit async pages: 1, from 0, to 1024 prio 0 00000008:00010000:3.0:1787827173.150422:0:27439:0:(osc_cache.c:565:osc_extent_merge()) ### extent: 000000001063fa34 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.150439:0:27439:0:(osc_cache.c:497:__osc_extent_remove()) ### extent: 000000001063fa34 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.150454:0:27439:0:(osc_cache.c:582:osc_extent_merge()) ### extent: 00000000df269729 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.150465:0:27439:0:(osc_cache.c:846:osc_extent_find()) ### extent: 00000000df269729 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.150483:0:27439:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff9e35d0e392f8, count 1, dirtied 1 00000080:00200000:3.0:1787827173.150488:0:27439:0:(vvp_io.c:1237:vvp_io_write_commit()) Committed 1 pages 1024 bytes, tot: 1024 00000080:00200000:3.0:1787827173.150493:0:27439:0:(vvp_io.c:1421:vvp_io_write_start()) oosfile: write bytes 0, result: 1024 00000020:00010000:3.0:1787827173.150503:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_release():245 00000020:00010000:3.0:1787827173.150507:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1787827173.150513:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c7d81d30 (ffff9e35da704cf8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1787827173.150516:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.150527:0:27439:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 17/0,0 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1787827173.150532:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_fini():72 00000020:00010000:3.0:1787827173.150534:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c7d81d30 (ffff9e35c614ce18/1) at cl_lock_fini():72 00000080:00200000:3.0:1787827173.150536:0:27439:0:(vvp_io.c:533:vvp_io_advance()) advancing 1024 bytes 00000080:00200000:3.0:1787827173.150539:0:27439:0:(file.c:2207:ll_file_io_generic()) Range unlock [421037, 421037] 00000080:00200000:3.0:1787827173.150543:0:27439:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787827173.150548:0:27439:0:(file.c:2240:ll_file_io_generic()) oosfile: 2 io complete with rc: 0, result: 1024, restart: 0 00000080:00200000:3.0:1787827173.150553:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724568576 i_blocks 3365288 00000080:00200000:3.0:1787827173.150558:0:27439:0:(file.c:2326:ll_file_io_generic()) iot: 2, result: 1024 00000080:00200008:3.0:1787827173.150567:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724568576, count: 1024 00000080:00200000:3.0:1787827173.150575:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421037 of 1024 to 1024 bytes 00000080:00200000:3.0:1787827173.150581:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724568576, len 1024, copied 1024 00000080:00200000:3.0:1787827173.150590:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724569600 i_blocks 3365288 00000080:00200000:3.0:1787827173.150598:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.150605:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724569600, count: 1024 00000080:00200000:3.0:1787827173.150611:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421037 of 2048 to 1024 bytes 00000080:00200000:3.0:1787827173.150617:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724569600, len 1024, copied 1024 00000080:00200000:3.0:1787827173.150625:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724570624 i_blocks 3365288 00000080:00200000:3.0:1787827173.150632:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.150638:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724570624, count: 1024 00000080:00200000:3.0:1787827173.150647:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421037 of 3072 to 1024 bytes 00000080:00200000:3.0:1787827173.150652:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724570624, len 1024, copied 1024 00000080:00200000:3.0:1787827173.150658:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724571648 i_blocks 3365288 00000080:00200000:3.0:1787827173.150661:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.150665:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724571648, count: 1024 00000080:00200000:3.0:1787827173.150672:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421038 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.150678:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 0, original count 1024 00000080:00200000:3.0:1787827173.150680:0:27439:0:(file.c:2090:ll_file_io_generic()) oosfile: write ppos: 1724571648, bytes: 1024 00000080:00200000:3.0:1787827173.150683:0:27439:0:(file.c:1848:ll_io_set_mirror()) oosfile: desiginated mirror: 0 00000020:00200000:3.0:1787827173.150687:0:27439:0:(cl_io.c:218:cl_io_rw_init()) header@000000007788600a[0x110, 368591, [0x200000402:0x27:0x0] hash] 00000020:00200000:3.0:1787827173.150697:0:27439:0:(cl_io.c:218:cl_io_rw_init()) io range: 2 [1724571648, 1724572672) 0 0 00000080:00200000:3.0:1787827173.150704:0:27439:0:(vvp_io.c:1889:vvp_io_init()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1787827173.150714:0:27439:0:(file.c:2170:ll_file_io_generic()) Range lock [421038, 421038] 00020000:00200000:3.0:1787827173.150720:0:27439:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 411 chunk: [1724571648, 1724572672) 1724572672, 1024 00020000:00200000:3.0:1787827173.150725:0:27439:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1787827173.150732:0:27439:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [421038, 421038] 00000008:00010000:3.0:1787827173.150741:0:27439:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1787827173.150744:0:27439:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff9e35c7d81d30, osc lock ffff9e35c2ddcc30, flags 0x0 00010000:00010000:3.0:1787827173.150753:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.150764:0:27439:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (860545024 860549119) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.150784:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 20/0,2 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.150795:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 20/0,2 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.150831:0:27439:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 19/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.150850:0:27439:0:(vvp_io.c:1339:vvp_io_write_start()) oosfile: write [1724571648, 1724572672) append 0 append lockpos 0 00000080:00200000:3.0:1787827173.150859:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421038 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.150879:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724571648, len 1024, copied 1024 00000080:00200000:3.0:1787827173.150887:0:27439:0:(rw26.c:933:ll_write_end()) page@00000000c98ca74a[3 000000005ce0a915 2 1 00000000a23cbd2a 1 0 421038] 00000080:00200000:3.0:1787827173.150893:0:27439:0:(rw26.c:933:ll_write_end()) vmpage @00000000d6d51b38 fffffc0001001 3:0 ffff9e34c3cb2488 421038 lru 00000080:00200000:3.0:1787827173.150900:0:27439:0:(rw26.c:933:ll_write_end()) osc-page@00000000f81161de 210094: 1< 0 - - > 2< 860545024 0 0 0x0 0x0 | 000000004db7037e 0000000052729811 > 3< 0 0 > 4< 0 0 8 483901440 - | - - + - > 5< - - + - | 0 - | 392 - -> 00000080:00200000:3.0:1787827173.150902:0:27439:0:(rw26.c:933:ll_write_end()) end page@00000000c98ca74a 00000080:00200000:3.0:1787827173.150903:0:27439:0:(rw26.c:933:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1787827173.150907:0:27439:0:(vvp_io.c:1214:vvp_io_write_commit()) commit async pages: 1, from 0, to 1024 prio 0 00000008:00010000:3.0:1787827173.150928:0:27439:0:(osc_cache.c:565:osc_extent_merge()) ### extent: 000000001063fa34 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.150953:0:27439:0:(osc_cache.c:497:__osc_extent_remove()) ### extent: 000000001063fa34 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.150966:0:27439:0:(osc_cache.c:582:osc_extent_merge()) ### extent: 00000000df269729 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.150976:0:27439:0:(osc_cache.c:846:osc_extent_find()) ### extent: 00000000df269729 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.150992:0:27439:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff9e35d0e392f8, count 1, dirtied 1 00000080:00200000:3.0:1787827173.150997:0:27439:0:(vvp_io.c:1237:vvp_io_write_commit()) Committed 1 pages 1024 bytes, tot: 1024 00000080:00200000:3.0:1787827173.150999:0:27439:0:(vvp_io.c:1421:vvp_io_write_start()) oosfile: write bytes 0, result: 1024 00000020:00010000:3.0:1787827173.151008:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_release():245 00000020:00010000:3.0:1787827173.151012:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1787827173.151016:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c7d81d30 (ffff9e35da704cf8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1787827173.151023:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.151037:0:27439:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 17/0,0 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1787827173.151053:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_fini():72 00000020:00010000:3.0:1787827173.151060:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c7d81d30 (ffff9e35c614ce18/1) at cl_lock_fini():72 00000080:00200000:3.0:1787827173.151063:0:27439:0:(vvp_io.c:533:vvp_io_advance()) advancing 1024 bytes 00000080:00200000:3.0:1787827173.151067:0:27439:0:(file.c:2207:ll_file_io_generic()) Range unlock [421038, 421038] 00000080:00200000:3.0:1787827173.151074:0:27439:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787827173.151077:0:27439:0:(file.c:2240:ll_file_io_generic()) oosfile: 2 io complete with rc: 0, result: 1024, restart: 0 00000080:00200000:3.0:1787827173.151084:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724572672 i_blocks 3365288 00000080:00200000:3.0:1787827173.151088:0:27439:0:(file.c:2326:ll_file_io_generic()) iot: 2, result: 1024 00000080:00200008:3.0:1787827173.151096:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724572672, count: 1024 00000080:00200000:3.0:1787827173.151102:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421038 of 1024 to 1024 bytes 00000080:00200000:3.0:1787827173.151106:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724572672, len 1024, copied 1024 00000080:00200000:3.0:1787827173.151112:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724573696 i_blocks 3365288 00000080:00200000:3.0:1787827173.151116:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.151120:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724573696, count: 1024 00000080:00200000:3.0:1787827173.151124:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421038 of 2048 to 1024 bytes 00000080:00200000:3.0:1787827173.151127:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724573696, len 1024, copied 1024 00000080:00200000:3.0:1787827173.151133:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724574720 i_blocks 3365288 00000080:00200000:3.0:1787827173.151138:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.151142:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724574720, count: 1024 00000080:00200000:3.0:1787827173.151144:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421038 of 3072 to 1024 bytes 00000080:00200000:3.0:1787827173.151146:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724574720, len 1024, copied 1024 00000080:00200000:3.0:1787827173.151152:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724575744 i_blocks 3365288 00000080:00200000:3.0:1787827173.151154:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.151158:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724575744, count: 1024 00000080:00200000:3.0:1787827173.151163:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421039 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.151166:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 0, original count 1024 00000080:00200000:3.0:1787827173.151167:0:27439:0:(file.c:2090:ll_file_io_generic()) oosfile: write ppos: 1724575744, bytes: 1024 00000080:00200000:3.0:1787827173.151169:0:27439:0:(file.c:1848:ll_io_set_mirror()) oosfile: desiginated mirror: 0 00000020:00200000:3.0:1787827173.151175:0:27439:0:(cl_io.c:218:cl_io_rw_init()) header@000000007788600a[0x110, 368592, [0x200000402:0x27:0x0] hash] 00000020:00200000:3.0:1787827173.151176:0:27439:0:(cl_io.c:218:cl_io_rw_init()) io range: 2 [1724575744, 1724576768) 0 0 00000080:00200000:3.0:1787827173.151179:0:27439:0:(vvp_io.c:1889:vvp_io_init()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1787827173.151186:0:27439:0:(file.c:2170:ll_file_io_generic()) Range lock [421039, 421039] 00020000:00200000:3.0:1787827173.151191:0:27439:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 411 chunk: [1724575744, 1724576768) 1724576768, 1024 00020000:00200000:3.0:1787827173.151193:0:27439:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1787827173.151197:0:27439:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [421039, 421039] 00000008:00010000:3.0:1787827173.151204:0:27439:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1787827173.151208:0:27439:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff9e35c7d81d30, osc lock ffff9e35c2ddcc30, flags 0x0 00010000:00010000:3.0:1787827173.151217:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.151232:0:27439:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (860549120 860553215) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.151248:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 20/0,2 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.151261:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 20/0,2 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.151275:0:27439:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 19/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.151289:0:27439:0:(vvp_io.c:1339:vvp_io_write_start()) oosfile: write [1724575744, 1724576768) append 0 append lockpos 0 00000080:00200000:3.0:1787827173.151297:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421039 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.151317:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724575744, len 1024, copied 1024 00000080:00200000:3.0:1787827173.151327:0:27439:0:(rw26.c:933:ll_write_end()) page@0000000099a4faba[3 000000005ce0a915 2 1 00000000a23cbd2a 1 0 421039] 00000080:00200000:3.0:1787827173.151333:0:27439:0:(rw26.c:933:ll_write_end()) vmpage @0000000027c7ea06 fffffc0001001 3:0 ffff9e34c3cb2bc8 421039 lru 00000080:00200000:3.0:1787827173.151340:0:27439:0:(rw26.c:933:ll_write_end()) osc-page@00000000c7914940 210095: 1< 0 - - > 2< 860549120 0 0 0x0 0x0 | 000000004db7037e 0000000052729811 > 3< 0 0 > 4< 0 0 8 483897344 - | - - + - > 5< - - + - | 0 - | 393 - -> 00000080:00200000:3.0:1787827173.151345:0:27439:0:(rw26.c:933:ll_write_end()) end page@0000000099a4faba 00000080:00200000:3.0:1787827173.151347:0:27439:0:(rw26.c:933:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1787827173.151350:0:27439:0:(vvp_io.c:1214:vvp_io_write_commit()) commit async pages: 1, from 0, to 1024 prio 0 00000008:00010000:3.0:1787827173.151363:0:27439:0:(osc_cache.c:565:osc_extent_merge()) ### extent: 000000001063fa34 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.151395:0:27439:0:(osc_cache.c:497:__osc_extent_remove()) ### extent: 000000001063fa34 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.151417:0:27439:0:(osc_cache.c:582:osc_extent_merge()) ### extent: 00000000df269729 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.151428:0:27439:0:(osc_cache.c:846:osc_extent_find()) ### extent: 00000000df269729 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.151447:0:27439:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff9e35d0e392f8, count 1, dirtied 1 00000080:00200000:3.0:1787827173.151453:0:27439:0:(vvp_io.c:1237:vvp_io_write_commit()) Committed 1 pages 1024 bytes, tot: 1024 00000080:00200000:3.0:1787827173.151458:0:27439:0:(vvp_io.c:1421:vvp_io_write_start()) oosfile: write bytes 0, result: 1024 00000020:00010000:3.0:1787827173.151468:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_release():245 00000020:00010000:3.0:1787827173.151472:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1787827173.151479:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c7d81d30 (ffff9e35da704cf8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1787827173.151487:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.151501:0:27439:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 17/0,0 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1787827173.151516:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_fini():72 00000020:00010000:3.0:1787827173.151521:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c7d81d30 (ffff9e35c614ce18/1) at cl_lock_fini():72 00000080:00200000:3.0:1787827173.151528:0:27439:0:(vvp_io.c:533:vvp_io_advance()) advancing 1024 bytes 00000080:00200000:3.0:1787827173.151533:0:27439:0:(file.c:2207:ll_file_io_generic()) Range unlock [421039, 421039] 00000080:00200000:3.0:1787827173.151540:0:27439:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787827173.151545:0:27439:0:(file.c:2240:ll_file_io_generic()) oosfile: 2 io complete with rc: 0, result: 1024, restart: 0 00000080:00200000:3.0:1787827173.151550:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724576768 i_blocks 3365288 00000080:00200000:3.0:1787827173.151555:0:27439:0:(file.c:2326:ll_file_io_generic()) iot: 2, result: 1024 00000080:00200008:3.0:1787827173.151567:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724576768, count: 1024 00000080:00200000:3.0:1787827173.151573:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421039 of 1024 to 1024 bytes 00000080:00200000:3.0:1787827173.151579:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724576768, len 1024, copied 1024 00000080:00200000:3.0:1787827173.151588:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724577792 i_blocks 3365288 00000080:00200000:3.0:1787827173.151593:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.151601:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724577792, count: 1024 00000080:00200000:3.0:1787827173.151607:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421039 of 2048 to 1024 bytes 00000080:00200000:3.0:1787827173.151614:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724577792, len 1024, copied 1024 00000080:00200000:3.0:1787827173.151623:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724578816 i_blocks 3365288 00000080:00200000:3.0:1787827173.151629:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.151636:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724578816, count: 1024 00000080:00200000:3.0:1787827173.151643:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421039 of 3072 to 1024 bytes 00000080:00200000:3.0:1787827173.151648:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724578816, len 1024, copied 1024 00000080:00200000:3.0:1787827173.151658:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724579840 i_blocks 3365288 00000080:00200000:3.0:1787827173.151663:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.151672:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724579840, count: 1024 00000080:00200000:3.0:1787827173.151681:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421040 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.151685:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 0, original count 1024 00000080:00200000:3.0:1787827173.151690:0:27439:0:(file.c:2090:ll_file_io_generic()) oosfile: write ppos: 1724579840, bytes: 1024 00000080:00200000:3.0:1787827173.151696:0:27439:0:(file.c:1848:ll_io_set_mirror()) oosfile: desiginated mirror: 0 00000020:00200000:3.0:1787827173.151706:0:27439:0:(cl_io.c:218:cl_io_rw_init()) header@000000007788600a[0x110, 368593, [0x200000402:0x27:0x0] hash] 00000020:00200000:3.0:1787827173.151709:0:27439:0:(cl_io.c:218:cl_io_rw_init()) io range: 2 [1724579840, 1724580864) 0 0 00000080:00200000:3.0:1787827173.151714:0:27439:0:(vvp_io.c:1889:vvp_io_init()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1787827173.151722:0:27439:0:(file.c:2170:ll_file_io_generic()) Range lock [421040, 421040] 00020000:00200000:3.0:1787827173.151727:0:27439:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 411 chunk: [1724579840, 1724580864) 1724580864, 1024 00020000:00200000:3.0:1787827173.151732:0:27439:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1787827173.151740:0:27439:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [421040, 421040] 00000008:00010000:3.0:1787827173.151752:0:27439:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1787827173.151756:0:27439:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff9e35c7d81d30, osc lock ffff9e35c2ddcc30, flags 0x0 00010000:00010000:3.0:1787827173.151768:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.151791:0:27439:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (860553216 860557311) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.151837:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 20/0,2 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.151856:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 20/0,2 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.151876:0:27439:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 19/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.151893:0:27439:0:(vvp_io.c:1339:vvp_io_write_start()) oosfile: write [1724579840, 1724580864) append 0 append lockpos 0 00000080:00200000:3.0:1787827173.151901:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421040 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.151932:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724579840, len 1024, copied 1024 00000080:00200000:3.0:1787827173.151938:0:27439:0:(rw26.c:933:ll_write_end()) page@000000000f03a23b[3 000000005ce0a915 2 1 00000000a23cbd2a 1 0 421040] 00000080:00200000:3.0:1787827173.151941:0:27439:0:(rw26.c:933:ll_write_end()) vmpage @000000008717bbe1 fffffc0001001 3:0 ffff9e34c3cb29f8 421040 lru 00000080:00200000:3.0:1787827173.151950:0:27439:0:(rw26.c:933:ll_write_end()) osc-page@000000003b443a85 210096: 1< 0 - - > 2< 860553216 0 0 0x0 0x0 | 000000004db7037e 0000000052729811 > 3< 0 0 > 4< 0 0 8 483893248 - | - - + - > 5< - - + - | 0 - | 394 - -> 00000080:00200000:3.0:1787827173.151956:0:27439:0:(rw26.c:933:ll_write_end()) end page@000000000f03a23b 00000080:00200000:3.0:1787827173.151959:0:27439:0:(rw26.c:933:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1787827173.151963:0:27439:0:(vvp_io.c:1214:vvp_io_write_commit()) commit async pages: 1, from 0, to 1024 prio 0 00000008:00010000:3.0:1787827173.151975:0:27439:0:(osc_cache.c:565:osc_extent_merge()) ### extent: 000000001063fa34 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.151996:0:27439:0:(osc_cache.c:497:__osc_extent_remove()) ### extent: 000000001063fa34 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.152020:0:27439:0:(osc_cache.c:582:osc_extent_merge()) ### extent: 00000000df269729 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.152043:0:27439:0:(osc_cache.c:846:osc_extent_find()) ### extent: 00000000df269729 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.152071:0:27439:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff9e35d0e392f8, count 1, dirtied 1 00000080:00200000:3.0:1787827173.152077:0:27439:0:(vvp_io.c:1237:vvp_io_write_commit()) Committed 1 pages 1024 bytes, tot: 1024 00000080:00200000:3.0:1787827173.152080:0:27439:0:(vvp_io.c:1421:vvp_io_write_start()) oosfile: write bytes 0, result: 1024 00000020:00010000:3.0:1787827173.152087:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_release():245 00000020:00010000:3.0:1787827173.152092:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1787827173.152097:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c7d81d30 (ffff9e35da704cf8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1787827173.152105:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.152118:0:27439:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 17/0,0 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1787827173.152135:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_fini():72 00000020:00010000:3.0:1787827173.152141:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c7d81d30 (ffff9e35c614ce18/1) at cl_lock_fini():72 00000080:00200000:3.0:1787827173.152149:0:27439:0:(vvp_io.c:533:vvp_io_advance()) advancing 1024 bytes 00000080:00200000:3.0:1787827173.152154:0:27439:0:(file.c:2207:ll_file_io_generic()) Range unlock [421040, 421040] 00000080:00200000:3.0:1787827173.152161:0:27439:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787827173.152165:0:27439:0:(file.c:2240:ll_file_io_generic()) oosfile: 2 io complete with rc: 0, result: 1024, restart: 0 00000080:00200000:3.0:1787827173.152173:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724580864 i_blocks 3365288 00000080:00200000:3.0:1787827173.152177:0:27439:0:(file.c:2326:ll_file_io_generic()) iot: 2, result: 1024 00000080:00200008:3.0:1787827173.152186:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724580864, count: 1024 00000080:00200000:3.0:1787827173.152197:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421040 of 1024 to 1024 bytes 00000080:00200000:3.0:1787827173.152204:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724580864, len 1024, copied 1024 00000080:00200000:3.0:1787827173.152213:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724581888 i_blocks 3365288 00000080:00200000:3.0:1787827173.152218:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.152227:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724581888, count: 1024 00000080:00200000:3.0:1787827173.152234:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421040 of 2048 to 1024 bytes 00000080:00200000:3.0:1787827173.152240:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724581888, len 1024, copied 1024 00000080:00200000:3.0:1787827173.152249:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724582912 i_blocks 3365288 00000080:00200000:3.0:1787827173.152256:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.152264:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724582912, count: 1024 00000080:00200000:3.0:1787827173.152272:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421040 of 3072 to 1024 bytes 00000080:00200000:3.0:1787827173.152280:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724582912, len 1024, copied 1024 00000080:00200000:3.0:1787827173.152291:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724583936 i_blocks 3365288 00000080:00200000:3.0:1787827173.152300:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.152307:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724583936, count: 1024 00000080:00200000:3.0:1787827173.152317:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421041 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.152322:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 0, original count 1024 00000080:00200000:3.0:1787827173.152328:0:27439:0:(file.c:2090:ll_file_io_generic()) oosfile: write ppos: 1724583936, bytes: 1024 00000080:00200000:3.0:1787827173.152334:0:27439:0:(file.c:1848:ll_io_set_mirror()) oosfile: desiginated mirror: 0 00000020:00200000:3.0:1787827173.152342:0:27439:0:(cl_io.c:218:cl_io_rw_init()) header@000000007788600a[0x110, 368594, [0x200000402:0x27:0x0] hash] 00000020:00200000:3.0:1787827173.152345:0:27439:0:(cl_io.c:218:cl_io_rw_init()) io range: 2 [1724583936, 1724584960) 0 0 00000080:00200000:3.0:1787827173.152350:0:27439:0:(vvp_io.c:1889:vvp_io_init()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1787827173.152359:0:27439:0:(file.c:2170:ll_file_io_generic()) Range lock [421041, 421041] 00020000:00200000:3.0:1787827173.152365:0:27439:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 411 chunk: [1724583936, 1724584960) 1724584960, 1024 00020000:00200000:3.0:1787827173.152370:0:27439:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1787827173.152378:0:27439:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [421041, 421041] 00000008:00010000:3.0:1787827173.152386:0:27439:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1787827173.152391:0:27439:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff9e35c7d81d30, osc lock ffff9e35c2ddcc30, flags 0x0 00010000:00010000:3.0:1787827173.152403:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.152417:0:27439:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (860557312 860561407) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.152427:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 20/0,2 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.152452:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 20/0,2 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.152466:0:27439:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 19/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.152485:0:27439:0:(vvp_io.c:1339:vvp_io_write_start()) oosfile: write [1724583936, 1724584960) append 0 append lockpos 0 00000080:00200000:3.0:1787827173.152494:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421041 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.152522:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724583936, len 1024, copied 1024 00000080:00200000:3.0:1787827173.152533:0:27439:0:(rw26.c:933:ll_write_end()) page@0000000064988c92[3 000000005ce0a915 2 1 00000000a23cbd2a 1 0 421041] 00000080:00200000:3.0:1787827173.152539:0:27439:0:(rw26.c:933:ll_write_end()) vmpage @000000009bbc2744 fffffc0001021 2:0 ffff9e34c3cb2ae0 421041 lru 00000080:00200000:3.0:1787827173.152553:0:27439:0:(rw26.c:933:ll_write_end()) osc-page@00000000d232fe8e 210097: 1< 0 - - > 2< 860557312 0 0 0x0 0x0 | 000000004db7037e 0000000052729811 > 3< 0 0 > 4< 0 0 8 483889152 - | - - + - > 5< - - + - | 0 - | 395 - -> 00000080:00200000:3.0:1787827173.152559:0:27439:0:(rw26.c:933:ll_write_end()) end page@0000000064988c92 00000080:00200000:3.0:1787827173.152563:0:27439:0:(rw26.c:933:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1787827173.152567:0:27439:0:(vvp_io.c:1214:vvp_io_write_commit()) commit async pages: 1, from 0, to 1024 prio 0 00000008:00010000:3.0:1787827173.152579:0:27439:0:(osc_cache.c:565:osc_extent_merge()) ### extent: 000000001063fa34 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.152608:0:27439:0:(osc_cache.c:497:__osc_extent_remove()) ### extent: 000000001063fa34 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.152627:0:27439:0:(osc_cache.c:582:osc_extent_merge()) ### extent: 00000000df269729 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.152644:0:27439:0:(osc_cache.c:846:osc_extent_find()) ### extent: 00000000df269729 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.152670:0:27439:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff9e35d0e392f8, count 1, dirtied 1 00000080:00200000:3.0:1787827173.152675:0:27439:0:(vvp_io.c:1237:vvp_io_write_commit()) Committed 1 pages 1024 bytes, tot: 1024 00000080:00200000:3.0:1787827173.152681:0:27439:0:(vvp_io.c:1421:vvp_io_write_start()) oosfile: write bytes 0, result: 1024 00000020:00010000:3.0:1787827173.152690:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_release():245 00000020:00010000:3.0:1787827173.152697:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1787827173.152701:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c7d81d30 (ffff9e35da704cf8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1787827173.152710:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.152735:0:27439:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 17/0,0 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1787827173.152758:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c633af38 (ffff9e35c614ce18/0) at cl_lock_fini():72 00000020:00010000:3.0:1787827173.152766:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c7d81d30 (ffff9e35c614ce18/1) at cl_lock_fini():72 00000080:00200000:3.0:1787827173.152773:0:27439:0:(vvp_io.c:533:vvp_io_advance()) advancing 1024 bytes 00000080:00200000:3.0:1787827173.152779:0:27439:0:(file.c:2207:ll_file_io_generic()) Range unlock [421041, 421041] 00000080:00200000:3.0:1787827173.152787:0:27439:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787827173.152791:0:27439:0:(file.c:2240:ll_file_io_generic()) oosfile: 2 io complete with rc: 0, result: 1024, restart: 0 00000080:00200000:3.0:1787827173.152818:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1724584960 i_blocks 3365288 00000080:00200000:3.0:1787827173.152822:0:27439:0:(file.c:2326:ll_file_io_generic()) iot: 2, result: 1024 00000080:00200008:3.0:1787827173.152833:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1724584960, count: 1024 00000080:00200000:3.0:1787827173.152841:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421041 of 1024 to 1024 bytes 00000080:00200000:3.0:1787827173.152847:0:27439:0:(rw26.c:875:ll_write_end()) pos 1724584960, len 1024, copied 1024 00000080:00200000:3.0:1787827173.599875:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728050176 i_blocks 3365288 00000080:00200000:3.0:1787827173.599886:0:27439:0:(file.c:2326:ll_file_io_generic()) iot: 2, result: 1024 00000080:00200008:3.0:1787827173.599898:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728050176, count: 1024 00000080:00200000:3.0:1787827173.599907:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421887 of 1024 to 1024 bytes 00000080:00200000:3.0:1787827173.599922:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728050176, len 1024, copied 1024 00000080:00200000:3.0:1787827173.599932:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728051200 i_blocks 3365288 00000080:00200000:3.0:1787827173.599937:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.599945:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728051200, count: 1024 00000080:00200000:3.0:1787827173.599950:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421887 of 2048 to 1024 bytes 00000080:00200000:3.0:1787827173.599954:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728051200, len 1024, copied 1024 00000080:00200000:3.0:1787827173.599960:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728052224 i_blocks 3365288 00000080:00200000:3.0:1787827173.599963:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.599968:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728052224, count: 1024 00000080:00200000:3.0:1787827173.599976:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421887 of 3072 to 1024 bytes 00000080:00200000:3.0:1787827173.599981:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728052224, len 1024, copied 1024 00000080:00200000:3.0:1787827173.599990:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728053248 i_blocks 3365288 00000080:00200000:3.0:1787827173.599997:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.600013:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728053248, count: 1024 00000080:00200000:3.0:1787827173.600019:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421888 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.600024:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 0, original count 1024 00000080:00200000:3.0:1787827173.600029:0:27439:0:(file.c:2090:ll_file_io_generic()) oosfile: write ppos: 1728053248, bytes: 1024 00000080:00200000:3.0:1787827173.600035:0:27439:0:(file.c:1848:ll_io_set_mirror()) oosfile: desiginated mirror: 0 00000020:00200000:3.0:1787827173.600049:0:27439:0:(cl_io.c:218:cl_io_rw_init()) header@000000007788600a[0x110, 369441, [0x200000402:0x27:0x0] hash] 00000020:00200000:3.0:1787827173.600052:0:27439:0:(cl_io.c:218:cl_io_rw_init()) io range: 2 [1728053248, 1728054272) 0 0 00000080:00200000:3.0:1787827173.600057:0:27439:0:(vvp_io.c:1889:vvp_io_init()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1787827173.600067:0:27439:0:(file.c:2170:ll_file_io_generic()) Range lock [421888, 421888] 00020000:00200000:3.0:1787827173.600072:0:27439:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 412 chunk: [1728053248, 1728054272) 1728054272, 1024 00020000:00200000:3.0:1787827173.600078:0:27439:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1787827173.600085:0:27439:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [421888, 421888] 00000008:00010000:3.0:1787827173.600093:0:27439:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1787827173.600097:0:27439:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff9e35c7d81d30, osc lock ffff9e35c2ddcc30, flags 0x0 00010000:00010000:3.0:1787827173.600108:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 19/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.600122:0:27439:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (864026624 864030719) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 19/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.600149:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 23/0,2 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.600172:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 23/0,2 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.600193:0:27439:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.600217:0:27439:0:(vvp_io.c:1339:vvp_io_write_start()) oosfile: write [1728053248, 1728054272) append 0 append lockpos 0 00000080:00200000:3.0:1787827173.600227:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421888 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.600253:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728053248, len 1024, copied 1024 00000080:00200000:3.0:1787827173.600269:0:27439:0:(rw26.c:933:ll_write_end()) page@00000000e08a1f30[3 000000005ce0a915 2 1 00000000d6bbf043 0 0 421888] 00000080:00200000:3.0:1787827173.600276:0:27439:0:(rw26.c:933:ll_write_end()) vmpage @000000004b4bd7c3 fffffc0001001 3:0 ffff9e34c3cb2828 421888 lru 00000080:00200000:3.0:1787827173.600288:0:27439:0:(rw26.c:933:ll_write_end()) osc-page@000000004e2a3858 210944: 1< 0 - - > 2< 864026624 0 0 0x0 0x0 | 00000000640821b9 0000000006873caa > 3< 0 0 > 4< 0 0 8 484638720 - | - - + - > 5< - - + - | 0 - | 17 - -> 00000080:00200000:3.0:1787827173.600294:0:27439:0:(rw26.c:933:ll_write_end()) end page@00000000e08a1f30 00000080:00200000:3.0:1787827173.600297:0:27439:0:(rw26.c:933:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1787827173.600302:0:27439:0:(vvp_io.c:1214:vvp_io_write_commit()) commit async pages: 1, from 0, to 1024 prio 0 00000008:00010000:3.0:1787827173.600321:0:27439:0:(osc_cache.c:858:osc_extent_find()) ### extent: 000000001063fa34 ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 21/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.600343:0:27439:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff9e35d0e392f8, count 1, dirtied 1 00000080:00200000:3.0:1787827173.600350:0:27439:0:(vvp_io.c:1237:vvp_io_write_commit()) Committed 1 pages 1024 bytes, tot: 1024 00000080:00200000:3.0:1787827173.600355:0:27439:0:(vvp_io.c:1421:vvp_io_write_start()) oosfile: write bytes 0, result: 1024 00000020:00010000:3.0:1787827173.600365:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_release():245 00000020:00010000:3.0:1787827173.600371:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1787827173.600375:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c7d81d30 (ffff9e35c614ce18/1) at cl_lock_cancel():142 00010000:00010000:3.0:1787827173.600382:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.600409:0:27439:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 21/0,0 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1787827173.600446:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_fini():72 00000020:00010000:3.0:1787827173.600454:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c7d81d30 (ffff9e35e04b4e18/1) at cl_lock_fini():72 00000080:00200000:3.0:1787827173.600463:0:27439:0:(vvp_io.c:533:vvp_io_advance()) advancing 1024 bytes 00000080:00200000:3.0:1787827173.600470:0:27439:0:(file.c:2207:ll_file_io_generic()) Range unlock [421888, 421888] 00000080:00200000:3.0:1787827173.600477:0:27439:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787827173.600484:0:27439:0:(file.c:2240:ll_file_io_generic()) oosfile: 2 io complete with rc: 0, result: 1024, restart: 0 00000080:00200000:3.0:1787827173.600498:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728054272 i_blocks 3365288 00000080:00200000:3.0:1787827173.600502:0:27439:0:(file.c:2326:ll_file_io_generic()) iot: 2, result: 1024 00000080:00200008:3.0:1787827173.600511:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728054272, count: 1024 00000080:00200000:3.0:1787827173.600520:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421888 of 1024 to 1024 bytes 00000080:00200000:3.0:1787827173.600526:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728054272, len 1024, copied 1024 00000080:00200000:3.0:1787827173.600535:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728055296 i_blocks 3365288 00000080:00200000:3.0:1787827173.600541:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.600556:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728055296, count: 1024 00000080:00200000:3.0:1787827173.600561:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421888 of 2048 to 1024 bytes 00000080:00200000:3.0:1787827173.600566:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728055296, len 1024, copied 1024 00000080:00200000:3.0:1787827173.600574:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728056320 i_blocks 3365288 00000080:00200000:3.0:1787827173.600580:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.600587:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728056320, count: 1024 00000080:00200000:3.0:1787827173.600592:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421888 of 3072 to 1024 bytes 00000080:00200000:3.0:1787827173.600598:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728056320, len 1024, copied 1024 00000080:00200000:3.0:1787827173.600606:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728057344 i_blocks 3365288 00000080:00200000:3.0:1787827173.600613:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.600619:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728057344, count: 1024 00000080:00200000:3.0:1787827173.600623:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421889 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.600625:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 0, original count 1024 00000080:00200000:3.0:1787827173.600631:0:27439:0:(file.c:2090:ll_file_io_generic()) oosfile: write ppos: 1728057344, bytes: 1024 00000080:00200000:3.0:1787827173.600635:0:27439:0:(file.c:1848:ll_io_set_mirror()) oosfile: desiginated mirror: 0 00000020:00200000:3.0:1787827173.600643:0:27439:0:(cl_io.c:218:cl_io_rw_init()) header@000000007788600a[0x110, 369442, [0x200000402:0x27:0x0] hash] 00000020:00200000:3.0:1787827173.600645:0:27439:0:(cl_io.c:218:cl_io_rw_init()) io range: 2 [1728057344, 1728058368) 0 0 00000080:00200000:3.0:1787827173.600650:0:27439:0:(vvp_io.c:1889:vvp_io_init()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1787827173.600660:0:27439:0:(file.c:2170:ll_file_io_generic()) Range lock [421889, 421889] 00020000:00200000:3.0:1787827173.600666:0:27439:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 412 chunk: [1728057344, 1728058368) 1728058368, 1024 00020000:00200000:3.0:1787827173.600672:0:27439:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1787827173.600682:0:27439:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [421889, 421889] 00000008:00010000:3.0:1787827173.600696:0:27439:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1787827173.600701:0:27439:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff9e35c7d81d30, osc lock ffff9e35c2ddcc30, flags 0x0 00010000:00010000:3.0:1787827173.600713:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 20/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.600730:0:27439:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (864030720 864034815) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 20/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.600756:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 24/0,2 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.600771:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 24/0,2 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.600780:0:27439:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 23/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.600787:0:27439:0:(vvp_io.c:1339:vvp_io_write_start()) oosfile: write [1728057344, 1728058368) append 0 append lockpos 0 00000080:00200000:3.0:1787827173.600795:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421889 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.600846:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728057344, len 1024, copied 1024 00000080:00200000:3.0:1787827173.600856:0:27439:0:(rw26.c:933:ll_write_end()) page@0000000018a25723[3 000000005ce0a915 2 1 00000000d6bbf043 0 0 421889] 00000080:00200000:3.0:1787827173.600862:0:27439:0:(rw26.c:933:ll_write_end()) vmpage @00000000db5db300 fffffc0001001 3:0 ffff9e34c3cb2570 421889 lru 00000080:00200000:3.0:1787827173.600876:0:27439:0:(rw26.c:933:ll_write_end()) osc-page@000000005f39eca1 210945: 1< 0 - - > 2< 864030720 0 0 0x0 0x0 | 00000000640821b9 0000000006873caa > 3< 0 0 > 4< 0 0 8 484610048 - | - - + - > 5< - - + - | 0 - | 18 - -> 00000080:00200000:3.0:1787827173.600880:0:27439:0:(rw26.c:933:ll_write_end()) end page@0000000018a25723 00000080:00200000:3.0:1787827173.600884:0:27439:0:(rw26.c:933:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1787827173.600888:0:27439:0:(vvp_io.c:1214:vvp_io_write_commit()) commit async pages: 1, from 0, to 1024 prio 0 00000008:00010000:3.0:1787827173.600902:0:27439:0:(osc_cache.c:565:osc_extent_merge()) ### extent: 00000000e070090a ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.600930:0:27439:0:(osc_cache.c:497:__osc_extent_remove()) ### extent: 00000000e070090a ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.600943:0:27439:0:(osc_cache.c:582:osc_extent_merge()) ### extent: 000000001063fa34 ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.600956:0:27439:0:(osc_cache.c:846:osc_extent_find()) ### extent: 000000001063fa34 ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.600971:0:27439:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff9e35d0e392f8, count 1, dirtied 1 00000080:00200000:3.0:1787827173.600975:0:27439:0:(vvp_io.c:1237:vvp_io_write_commit()) Committed 1 pages 1024 bytes, tot: 1024 00000080:00200000:3.0:1787827173.600979:0:27439:0:(vvp_io.c:1421:vvp_io_write_start()) oosfile: write bytes 0, result: 1024 00000020:00010000:3.0:1787827173.600988:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_release():245 00000020:00010000:3.0:1787827173.600992:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1787827173.601000:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c7d81d30 (ffff9e35c614ce18/1) at cl_lock_cancel():142 00010000:00010000:3.0:1787827173.601008:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.601029:0:27439:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 21/0,0 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1787827173.601053:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_fini():72 00000020:00010000:3.0:1787827173.601058:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c7d81d30 (ffff9e35e04b4e18/1) at cl_lock_fini():72 00000080:00200000:3.0:1787827173.601066:0:27439:0:(vvp_io.c:533:vvp_io_advance()) advancing 1024 bytes 00000080:00200000:3.0:1787827173.601070:0:27439:0:(file.c:2207:ll_file_io_generic()) Range unlock [421889, 421889] 00000080:00200000:3.0:1787827173.601074:0:27439:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787827173.601081:0:27439:0:(file.c:2240:ll_file_io_generic()) oosfile: 2 io complete with rc: 0, result: 1024, restart: 0 00000080:00200000:3.0:1787827173.601090:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728058368 i_blocks 3365288 00000080:00200000:3.0:1787827173.601092:0:27439:0:(file.c:2326:ll_file_io_generic()) iot: 2, result: 1024 00000080:00200008:3.0:1787827173.601100:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728058368, count: 1024 00000080:00200000:3.0:1787827173.601106:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421889 of 1024 to 1024 bytes 00000080:00200000:3.0:1787827173.601108:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728058368, len 1024, copied 1024 00000080:00200000:3.0:1787827173.601112:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728059392 i_blocks 3365288 00000080:00200000:3.0:1787827173.601113:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.601116:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728059392, count: 1024 00000080:00200000:3.0:1787827173.601118:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421889 of 2048 to 1024 bytes 00000080:00200000:3.0:1787827173.601121:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728059392, len 1024, copied 1024 00000080:00200000:3.0:1787827173.601127:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728060416 i_blocks 3365288 00000080:00200000:3.0:1787827173.601131:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.601135:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728060416, count: 1024 00000080:00200000:3.0:1787827173.601142:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421889 of 3072 to 1024 bytes 00000080:00200000:3.0:1787827173.601147:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728060416, len 1024, copied 1024 00000080:00200000:3.0:1787827173.601153:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728061440 i_blocks 3365288 00000080:00200000:3.0:1787827173.601159:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.601166:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728061440, count: 1024 00000080:00200000:3.0:1787827173.601172:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421890 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.601177:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 0, original count 1024 00000080:00200000:3.0:1787827173.601181:0:27439:0:(file.c:2090:ll_file_io_generic()) oosfile: write ppos: 1728061440, bytes: 1024 00000080:00200000:3.0:1787827173.601187:0:27439:0:(file.c:1848:ll_io_set_mirror()) oosfile: desiginated mirror: 0 00000020:00200000:3.0:1787827173.601193:0:27439:0:(cl_io.c:218:cl_io_rw_init()) header@000000007788600a[0x110, 369443, [0x200000402:0x27:0x0] hash] 00000020:00200000:3.0:1787827173.601196:0:27439:0:(cl_io.c:218:cl_io_rw_init()) io range: 2 [1728061440, 1728062464) 0 0 00000080:00200000:3.0:1787827173.601201:0:27439:0:(vvp_io.c:1889:vvp_io_init()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1787827173.601211:0:27439:0:(file.c:2170:ll_file_io_generic()) Range lock [421890, 421890] 00020000:00200000:3.0:1787827173.601217:0:27439:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 412 chunk: [1728061440, 1728062464) 1728062464, 1024 00020000:00200000:3.0:1787827173.601222:0:27439:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1787827173.601228:0:27439:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [421890, 421890] 00000008:00010000:3.0:1787827173.601237:0:27439:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1787827173.601241:0:27439:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff9e35c7d81d30, osc lock ffff9e35c2ddcc30, flags 0x0 00010000:00010000:3.0:1787827173.601252:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 20/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.601270:0:27439:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (864034816 864038911) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 20/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.601284:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 24/0,2 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.601316:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 24/0,2 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.601332:0:27439:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 23/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.601347:0:27439:0:(vvp_io.c:1339:vvp_io_write_start()) oosfile: write [1728061440, 1728062464) append 0 append lockpos 0 00000080:00200000:3.0:1787827173.601356:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421890 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.601377:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728061440, len 1024, copied 1024 00000080:00200000:3.0:1787827173.601384:0:27439:0:(rw26.c:933:ll_write_end()) page@000000001e9f8534[3 000000005ce0a915 2 1 00000000d6bbf043 0 0 421890] 00000080:00200000:3.0:1787827173.601390:0:27439:0:(rw26.c:933:ll_write_end()) vmpage @00000000a2fd2a1b fffffc0001001 3:0 ffff9e34c3cb2e80 421890 lru 00000080:00200000:3.0:1787827173.601400:0:27439:0:(rw26.c:933:ll_write_end()) osc-page@00000000181fa495 210946: 1< 0 - - > 2< 864034816 0 0 0x0 0x0 | 00000000640821b9 0000000006873caa > 3< 0 0 > 4< 0 0 8 484605952 - | - - + - > 5< - - + - | 0 - | 19 - -> 00000080:00200000:3.0:1787827173.601406:0:27439:0:(rw26.c:933:ll_write_end()) end page@000000001e9f8534 00000080:00200000:3.0:1787827173.601409:0:27439:0:(rw26.c:933:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1787827173.601414:0:27439:0:(vvp_io.c:1214:vvp_io_write_commit()) commit async pages: 1, from 0, to 1024 prio 0 00000008:00010000:3.0:1787827173.601424:0:27439:0:(osc_cache.c:565:osc_extent_merge()) ### extent: 00000000e070090a ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.601435:0:27439:0:(osc_cache.c:497:__osc_extent_remove()) ### extent: 00000000e070090a ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.601448:0:27439:0:(osc_cache.c:582:osc_extent_merge()) ### extent: 000000001063fa34 ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.601463:0:27439:0:(osc_cache.c:846:osc_extent_find()) ### extent: 000000001063fa34 ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.601482:0:27439:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff9e35d0e392f8, count 1, dirtied 1 00000080:00200000:3.0:1787827173.601487:0:27439:0:(vvp_io.c:1237:vvp_io_write_commit()) Committed 1 pages 1024 bytes, tot: 1024 00000080:00200000:3.0:1787827173.601490:0:27439:0:(vvp_io.c:1421:vvp_io_write_start()) oosfile: write bytes 0, result: 1024 00000020:00010000:3.0:1787827173.601497:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_release():245 00000020:00010000:3.0:1787827173.601500:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1787827173.601506:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c7d81d30 (ffff9e35c614ce18/1) at cl_lock_cancel():142 00010000:00010000:3.0:1787827173.601512:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.601519:0:27439:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 21/0,0 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1787827173.601533:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_fini():72 00000020:00010000:3.0:1787827173.601536:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c7d81d30 (ffff9e35e04b4e18/1) at cl_lock_fini():72 00000080:00200000:3.0:1787827173.601539:0:27439:0:(vvp_io.c:533:vvp_io_advance()) advancing 1024 bytes 00000080:00200000:3.0:1787827173.601543:0:27439:0:(file.c:2207:ll_file_io_generic()) Range unlock [421890, 421890] 00000080:00200000:3.0:1787827173.601546:0:27439:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787827173.601551:0:27439:0:(file.c:2240:ll_file_io_generic()) oosfile: 2 io complete with rc: 0, result: 1024, restart: 0 00000080:00200000:3.0:1787827173.601557:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728062464 i_blocks 3365288 00000080:00200000:3.0:1787827173.601560:0:27439:0:(file.c:2326:ll_file_io_generic()) iot: 2, result: 1024 00000080:00200008:3.0:1787827173.601566:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728062464, count: 1024 00000080:00200000:3.0:1787827173.601574:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421890 of 1024 to 1024 bytes 00000080:00200000:3.0:1787827173.601578:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728062464, len 1024, copied 1024 00000080:00200000:3.0:1787827173.601583:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728063488 i_blocks 3365288 00000080:00200000:3.0:1787827173.601587:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.601591:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728063488, count: 1024 00000080:00200000:3.0:1787827173.601596:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421890 of 2048 to 1024 bytes 00000080:00200000:3.0:1787827173.601601:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728063488, len 1024, copied 1024 00000080:00200000:3.0:1787827173.601607:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728064512 i_blocks 3365288 00000080:00200000:3.0:1787827173.601611:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.601619:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728064512, count: 1024 00000080:00200000:3.0:1787827173.601628:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421890 of 3072 to 1024 bytes 00000080:00200000:3.0:1787827173.601633:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728064512, len 1024, copied 1024 00000080:00200000:3.0:1787827173.601641:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728065536 i_blocks 3365288 00000080:00200000:3.0:1787827173.601647:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.601654:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728065536, count: 1024 00000080:00200000:3.0:1787827173.601660:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421891 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.601666:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 0, original count 1024 00000080:00200000:3.0:1787827173.601671:0:27439:0:(file.c:2090:ll_file_io_generic()) oosfile: write ppos: 1728065536, bytes: 1024 00000080:00200000:3.0:1787827173.601674:0:27439:0:(file.c:1848:ll_io_set_mirror()) oosfile: desiginated mirror: 0 00000020:00200000:3.0:1787827173.601678:0:27439:0:(cl_io.c:218:cl_io_rw_init()) header@000000007788600a[0x110, 369444, [0x200000402:0x27:0x0] hash] 00000020:00200000:3.0:1787827173.601680:0:27439:0:(cl_io.c:218:cl_io_rw_init()) io range: 2 [1728065536, 1728066560) 0 0 00000080:00200000:3.0:1787827173.601682:0:27439:0:(vvp_io.c:1889:vvp_io_init()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1787827173.601687:0:27439:0:(file.c:2170:ll_file_io_generic()) Range lock [421891, 421891] 00020000:00200000:3.0:1787827173.601691:0:27439:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 412 chunk: [1728065536, 1728066560) 1728066560, 1024 00020000:00200000:3.0:1787827173.601697:0:27439:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1787827173.601705:0:27439:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [421891, 421891] 00000008:00010000:3.0:1787827173.601716:0:27439:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1787827173.601721:0:27439:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff9e35c7d81d30, osc lock ffff9e35c2ddcc30, flags 0x0 00010000:00010000:3.0:1787827173.601733:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 20/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.601743:0:27439:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (864038912 864043007) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 20/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.601761:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 24/0,2 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.601789:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 24/0,2 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.601812:0:27439:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 23/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.601826:0:27439:0:(vvp_io.c:1339:vvp_io_write_start()) oosfile: write [1728065536, 1728066560) append 0 append lockpos 0 00000080:00200000:3.0:1787827173.601832:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421891 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.601847:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728065536, len 1024, copied 1024 00000080:00200000:3.0:1787827173.601853:0:27439:0:(rw26.c:933:ll_write_end()) page@00000000d4490a56[3 000000005ce0a915 2 1 00000000d6bbf043 0 0 421891] 00000080:00200000:3.0:1787827173.601855:0:27439:0:(rw26.c:933:ll_write_end()) vmpage @00000000adbec7ee fffffc0001001 3:0 ffff9e34c3cb2740 421891 lru 00000080:00200000:3.0:1787827173.601864:0:27439:0:(rw26.c:933:ll_write_end()) osc-page@000000003894892a 210947: 1< 0 - - > 2< 864038912 0 0 0x0 0x0 | 00000000640821b9 0000000006873caa > 3< 0 0 > 4< 0 0 8 484601856 - | - - + - > 5< - - + - | 0 - | 20 - -> 00000080:00200000:3.0:1787827173.601868:0:27439:0:(rw26.c:933:ll_write_end()) end page@00000000d4490a56 00000080:00200000:3.0:1787827173.601870:0:27439:0:(rw26.c:933:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1787827173.601874:0:27439:0:(vvp_io.c:1214:vvp_io_write_commit()) commit async pages: 1, from 0, to 1024 prio 0 00000008:00010000:3.0:1787827173.601882:0:27439:0:(osc_cache.c:565:osc_extent_merge()) ### extent: 00000000e070090a ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.601901:0:27439:0:(osc_cache.c:497:__osc_extent_remove()) ### extent: 00000000e070090a ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.601935:0:27439:0:(osc_cache.c:582:osc_extent_merge()) ### extent: 000000001063fa34 ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.601943:0:27439:0:(osc_cache.c:846:osc_extent_find()) ### extent: 000000001063fa34 ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.601958:0:27439:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff9e35d0e392f8, count 1, dirtied 1 00000080:00200000:3.0:1787827173.601962:0:27439:0:(vvp_io.c:1237:vvp_io_write_commit()) Committed 1 pages 1024 bytes, tot: 1024 00000080:00200000:3.0:1787827173.601966:0:27439:0:(vvp_io.c:1421:vvp_io_write_start()) oosfile: write bytes 0, result: 1024 00000020:00010000:3.0:1787827173.601972:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_release():245 00000020:00010000:3.0:1787827173.601976:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1787827173.601980:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c7d81d30 (ffff9e35c614ce18/1) at cl_lock_cancel():142 00010000:00010000:3.0:1787827173.601986:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.601994:0:27439:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 21/0,0 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1787827173.602009:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_fini():72 00000020:00010000:3.0:1787827173.602016:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c7d81d30 (ffff9e35e04b4e18/1) at cl_lock_fini():72 00000080:00200000:3.0:1787827173.602021:0:27439:0:(vvp_io.c:533:vvp_io_advance()) advancing 1024 bytes 00000080:00200000:3.0:1787827173.602027:0:27439:0:(file.c:2207:ll_file_io_generic()) Range unlock [421891, 421891] 00000080:00200000:3.0:1787827173.602034:0:27439:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787827173.602043:0:27439:0:(file.c:2240:ll_file_io_generic()) oosfile: 2 io complete with rc: 0, result: 1024, restart: 0 00000080:00200000:3.0:1787827173.602101:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728066560 i_blocks 3365288 00000080:00200000:3.0:1787827173.602106:0:27439:0:(file.c:2326:ll_file_io_generic()) iot: 2, result: 1024 00000080:00200008:3.0:1787827173.602118:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728066560, count: 1024 00000080:00200000:3.0:1787827173.602128:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421891 of 1024 to 1024 bytes 00000080:00200000:3.0:1787827173.602135:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728066560, len 1024, copied 1024 00000080:00200000:3.0:1787827173.602145:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728067584 i_blocks 3365288 00000080:00200000:3.0:1787827173.602150:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.602157:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728067584, count: 1024 00000080:00200000:3.0:1787827173.602164:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421891 of 2048 to 1024 bytes 00000080:00200000:3.0:1787827173.602170:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728067584, len 1024, copied 1024 00000080:00200000:3.0:1787827173.602177:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728068608 i_blocks 3365288 00000080:00200000:3.0:1787827173.602182:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.602188:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728068608, count: 1024 00000080:00200000:3.0:1787827173.602194:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421891 of 3072 to 1024 bytes 00000080:00200000:3.0:1787827173.602199:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728068608, len 1024, copied 1024 00000080:00200000:3.0:1787827173.602206:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728069632 i_blocks 3365288 00000080:00200000:3.0:1787827173.602213:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.602220:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728069632, count: 1024 00000080:00200000:3.0:1787827173.602226:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421892 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.602230:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 0, original count 1024 00000080:00200000:3.0:1787827173.602235:0:27439:0:(file.c:2090:ll_file_io_generic()) oosfile: write ppos: 1728069632, bytes: 1024 00000080:00200000:3.0:1787827173.602239:0:27439:0:(file.c:1848:ll_io_set_mirror()) oosfile: desiginated mirror: 0 00000020:00200000:3.0:1787827173.602249:0:27439:0:(cl_io.c:218:cl_io_rw_init()) header@000000007788600a[0x110, 369445, [0x200000402:0x27:0x0] hash] 00000020:00200000:3.0:1787827173.602251:0:27439:0:(cl_io.c:218:cl_io_rw_init()) io range: 2 [1728069632, 1728070656) 0 0 00000080:00200000:3.0:1787827173.602256:0:27439:0:(vvp_io.c:1889:vvp_io_init()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1787827173.602267:0:27439:0:(file.c:2170:ll_file_io_generic()) Range lock [421892, 421892] 00020000:00200000:3.0:1787827173.602273:0:27439:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 412 chunk: [1728069632, 1728070656) 1728070656, 1024 00020000:00200000:3.0:1787827173.602277:0:27439:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1787827173.602286:0:27439:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [421892, 421892] 00000008:00010000:3.0:1787827173.602294:0:27439:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1787827173.602297:0:27439:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff9e35c7d81d30, osc lock ffff9e35c2ddcc30, flags 0x0 00010000:00010000:3.0:1787827173.602306:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 20/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.602320:0:27439:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (864043008 864047103) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 20/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.602333:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 24/0,2 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.602352:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 24/0,2 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.602359:0:27439:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 23/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.602366:0:27439:0:(vvp_io.c:1339:vvp_io_write_start()) oosfile: write [1728069632, 1728070656) append 0 append lockpos 0 00000080:00200000:3.0:1787827173.602375:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421892 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.602395:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728069632, len 1024, copied 1024 00000080:00200000:3.0:1787827173.602406:0:27439:0:(rw26.c:933:ll_write_end()) page@00000000606bb0a7[3 000000005ce0a915 2 1 00000000d6bbf043 0 0 421892] 00000080:00200000:3.0:1787827173.602414:0:27439:0:(rw26.c:933:ll_write_end()) vmpage @0000000002da675a fffffc0001001 3:0 ffff9e34c3cb2910 421892 lru 00000080:00200000:3.0:1787827173.602422:0:27439:0:(rw26.c:933:ll_write_end()) osc-page@00000000699d7f58 210948: 1< 0 - - > 2< 864043008 0 0 0x0 0x0 | 00000000640821b9 0000000006873caa > 3< 0 0 > 4< 0 0 8 484597760 - | - - + - > 5< - - + - | 0 - | 21 - -> 00000080:00200000:3.0:1787827173.602427:0:27439:0:(rw26.c:933:ll_write_end()) end page@00000000606bb0a7 00000080:00200000:3.0:1787827173.602429:0:27439:0:(rw26.c:933:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1787827173.602435:0:27439:0:(vvp_io.c:1214:vvp_io_write_commit()) commit async pages: 1, from 0, to 1024 prio 0 00000008:00010000:3.0:1787827173.602448:0:27439:0:(osc_cache.c:565:osc_extent_merge()) ### extent: 00000000e070090a ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.602467:0:27439:0:(osc_cache.c:497:__osc_extent_remove()) ### extent: 00000000e070090a ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.602488:0:27439:0:(osc_cache.c:582:osc_extent_merge()) ### extent: 000000001063fa34 ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.602508:0:27439:0:(osc_cache.c:846:osc_extent_find()) ### extent: 000000001063fa34 ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.602521:0:27439:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff9e35d0e392f8, count 1, dirtied 1 00000080:00200000:3.0:1787827173.602525:0:27439:0:(vvp_io.c:1237:vvp_io_write_commit()) Committed 1 pages 1024 bytes, tot: 1024 00000080:00200000:3.0:1787827173.602528:0:27439:0:(vvp_io.c:1421:vvp_io_write_start()) oosfile: write bytes 0, result: 1024 00000020:00010000:3.0:1787827173.602535:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_release():245 00000020:00010000:3.0:1787827173.602540:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1787827173.602545:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c7d81d30 (ffff9e35c614ce18/1) at cl_lock_cancel():142 00010000:00010000:3.0:1787827173.602549:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.602560:0:27439:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 21/0,0 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1787827173.602574:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_fini():72 00000020:00010000:3.0:1787827173.602577:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c7d81d30 (ffff9e35e04b4e18/1) at cl_lock_fini():72 00000080:00200000:3.0:1787827173.602583:0:27439:0:(vvp_io.c:533:vvp_io_advance()) advancing 1024 bytes 00000080:00200000:3.0:1787827173.602588:0:27439:0:(file.c:2207:ll_file_io_generic()) Range unlock [421892, 421892] 00000080:00200000:3.0:1787827173.602593:0:27439:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787827173.602599:0:27439:0:(file.c:2240:ll_file_io_generic()) oosfile: 2 io complete with rc: 0, result: 1024, restart: 0 00000080:00200000:3.0:1787827173.602607:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728070656 i_blocks 3365288 00000080:00200000:3.0:1787827173.602612:0:27439:0:(file.c:2326:ll_file_io_generic()) iot: 2, result: 1024 00000080:00200008:3.0:1787827173.602620:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728070656, count: 1024 00000080:00200000:3.0:1787827173.602627:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421892 of 1024 to 1024 bytes 00000080:00200000:3.0:1787827173.602631:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728070656, len 1024, copied 1024 00000080:00200000:3.0:1787827173.602638:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728071680 i_blocks 3365288 00000080:00200000:3.0:1787827173.602643:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.602648:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728071680, count: 1024 00000080:00200000:3.0:1787827173.602657:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421892 of 2048 to 1024 bytes 00000080:00200000:3.0:1787827173.602661:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728071680, len 1024, copied 1024 00000080:00200000:3.0:1787827173.602668:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728072704 i_blocks 3365288 00000080:00200000:3.0:1787827173.602673:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.602677:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728072704, count: 1024 00000080:00200000:3.0:1787827173.602681:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421892 of 3072 to 1024 bytes 00000080:00200000:3.0:1787827173.602687:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728072704, len 1024, copied 1024 00000080:00200000:3.0:1787827173.602692:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728073728 i_blocks 3365288 00000080:00200000:3.0:1787827173.602697:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.602701:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728073728, count: 1024 00000080:00200000:3.0:1787827173.602710:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421893 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.602715:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 0, original count 1024 00000080:00200000:3.0:1787827173.602720:0:27439:0:(file.c:2090:ll_file_io_generic()) oosfile: write ppos: 1728073728, bytes: 1024 00000080:00200000:3.0:1787827173.602726:0:27439:0:(file.c:1848:ll_io_set_mirror()) oosfile: desiginated mirror: 0 00000020:00200000:3.0:1787827173.602733:0:27439:0:(cl_io.c:218:cl_io_rw_init()) header@000000007788600a[0x110, 369446, [0x200000402:0x27:0x0] hash] 00000020:00200000:3.0:1787827173.602737:0:27439:0:(cl_io.c:218:cl_io_rw_init()) io range: 2 [1728073728, 1728074752) 0 0 00000080:00200000:3.0:1787827173.602743:0:27439:0:(vvp_io.c:1889:vvp_io_init()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1787827173.602749:0:27439:0:(file.c:2170:ll_file_io_generic()) Range lock [421893, 421893] 00020000:00200000:3.0:1787827173.602755:0:27439:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 412 chunk: [1728073728, 1728074752) 1728074752, 1024 00020000:00200000:3.0:1787827173.602758:0:27439:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1787827173.602765:0:27439:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [421893, 421893] 00000008:00010000:3.0:1787827173.602771:0:27439:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1787827173.602774:0:27439:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff9e35c7d81d30, osc lock ffff9e35c2ddcc30, flags 0x0 00010000:00010000:3.0:1787827173.602781:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 20/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.602788:0:27439:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (864047104 864051199) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 20/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.602795:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 24/0,2 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.602828:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 24/0,2 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.602839:0:27439:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 23/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.602852:0:27439:0:(vvp_io.c:1339:vvp_io_write_start()) oosfile: write [1728073728, 1728074752) append 0 append lockpos 0 00000080:00200000:3.0:1787827173.602859:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421893 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.602874:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728073728, len 1024, copied 1024 00000080:00200000:3.0:1787827173.602880:0:27439:0:(rw26.c:933:ll_write_end()) page@000000009679b8e5[3 000000005ce0a915 2 1 00000000d6bbf043 0 0 421893] 00000080:00200000:3.0:1787827173.602887:0:27439:0:(rw26.c:933:ll_write_end()) vmpage @00000000675b38b6 fffffc0001001 3:0 ffff9e34c3cb20e8 421893 lru 00000080:00200000:3.0:1787827173.602899:0:27439:0:(rw26.c:933:ll_write_end()) osc-page@00000000fd817abe 210949: 1< 0 - - > 2< 864047104 0 0 0x0 0x0 | 00000000640821b9 0000000006873caa > 3< 0 0 > 4< 0 0 8 484593664 - | - - + - > 5< - - + - | 0 - | 22 - -> 00000080:00200000:3.0:1787827173.602904:0:27439:0:(rw26.c:933:ll_write_end()) end page@000000009679b8e5 00000080:00200000:3.0:1787827173.602906:0:27439:0:(rw26.c:933:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1787827173.602911:0:27439:0:(vvp_io.c:1214:vvp_io_write_commit()) commit async pages: 1, from 0, to 1024 prio 0 00000008:00010000:3.0:1787827173.602936:0:27439:0:(osc_cache.c:565:osc_extent_merge()) ### extent: 00000000e070090a ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.602954:0:27439:0:(osc_cache.c:497:__osc_extent_remove()) ### extent: 00000000e070090a ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.602963:0:27439:0:(osc_cache.c:582:osc_extent_merge()) ### extent: 000000001063fa34 ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.602979:0:27439:0:(osc_cache.c:846:osc_extent_find()) ### extent: 000000001063fa34 ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.603004:0:27439:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff9e35d0e392f8, count 1, dirtied 1 00000080:00200000:3.0:1787827173.603011:0:27439:0:(vvp_io.c:1237:vvp_io_write_commit()) Committed 1 pages 1024 bytes, tot: 1024 00000080:00200000:3.0:1787827173.603017:0:27439:0:(vvp_io.c:1421:vvp_io_write_start()) oosfile: write bytes 0, result: 1024 00000020:00010000:3.0:1787827173.603028:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_release():245 00000020:00010000:3.0:1787827173.603035:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1787827173.603039:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c7d81d30 (ffff9e35c614ce18/1) at cl_lock_cancel():142 00010000:00010000:3.0:1787827173.603045:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.603053:0:27439:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 21/0,0 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1787827173.603070:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_fini():72 00000020:00010000:3.0:1787827173.603076:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c7d81d30 (ffff9e35e04b4e18/1) at cl_lock_fini():72 00000080:00200000:3.0:1787827173.603085:0:27439:0:(vvp_io.c:533:vvp_io_advance()) advancing 1024 bytes 00000080:00200000:3.0:1787827173.603090:0:27439:0:(file.c:2207:ll_file_io_generic()) Range unlock [421893, 421893] 00000080:00200000:3.0:1787827173.603197:0:27439:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787827173.603206:0:27439:0:(file.c:2240:ll_file_io_generic()) oosfile: 2 io complete with rc: 0, result: 1024, restart: 0 00000080:00200000:3.0:1787827173.603212:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728074752 i_blocks 3365288 00000080:00200000:3.0:1787827173.603219:0:27439:0:(file.c:2326:ll_file_io_generic()) iot: 2, result: 1024 00000080:00200008:3.0:1787827173.603229:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728074752, count: 1024 00000080:00200000:3.0:1787827173.603236:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421893 of 1024 to 1024 bytes 00000080:00200000:3.0:1787827173.603242:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728074752, len 1024, copied 1024 00000080:00200000:3.0:1787827173.603291:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728075776 i_blocks 3365288 00000080:00200000:3.0:1787827173.603297:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.603307:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728075776, count: 1024 00000080:00200000:3.0:1787827173.603316:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421893 of 2048 to 1024 bytes 00000080:00200000:3.0:1787827173.603322:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728075776, len 1024, copied 1024 00000080:00200000:3.0:1787827173.603328:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728076800 i_blocks 3365288 00000080:00200000:3.0:1787827173.603331:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.603336:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728076800, count: 1024 00000080:00200000:3.0:1787827173.603338:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421893 of 3072 to 1024 bytes 00000080:00200000:3.0:1787827173.603340:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728076800, len 1024, copied 1024 00000080:00200000:3.0:1787827173.603344:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728077824 i_blocks 3365288 00000080:00200000:3.0:1787827173.603349:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.603354:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728077824, count: 1024 00000080:00200000:3.0:1787827173.603360:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421894 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.603363:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 0, original count 1024 00000080:00200000:3.0:1787827173.603367:0:27439:0:(file.c:2090:ll_file_io_generic()) oosfile: write ppos: 1728077824, bytes: 1024 00000080:00200000:3.0:1787827173.603373:0:27439:0:(file.c:1848:ll_io_set_mirror()) oosfile: desiginated mirror: 0 00000020:00200000:3.0:1787827173.603383:0:27439:0:(cl_io.c:218:cl_io_rw_init()) header@000000007788600a[0x110, 369447, [0x200000402:0x27:0x0] hash] 00000020:00200000:3.0:1787827173.603388:0:27439:0:(cl_io.c:218:cl_io_rw_init()) io range: 2 [1728077824, 1728078848) 0 0 00000080:00200000:3.0:1787827173.603392:0:27439:0:(vvp_io.c:1889:vvp_io_init()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1787827173.603403:0:27439:0:(file.c:2170:ll_file_io_generic()) Range lock [421894, 421894] 00020000:00200000:3.0:1787827173.603408:0:27439:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 412 chunk: [1728077824, 1728078848) 1728078848, 1024 00020000:00200000:3.0:1787827173.603414:0:27439:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1787827173.603423:0:27439:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [421894, 421894] 00000008:00010000:3.0:1787827173.603431:0:27439:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1787827173.603436:0:27439:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff9e35c7d81d30, osc lock ffff9e35c2ddcc30, flags 0x0 00010000:00010000:3.0:1787827173.603448:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 20/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.603466:0:27439:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (864051200 864055295) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 20/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.603480:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 24/0,2 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.603500:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 24/0,2 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.603513:0:27439:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 23/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.603528:0:27439:0:(vvp_io.c:1339:vvp_io_write_start()) oosfile: write [1728077824, 1728078848) append 0 append lockpos 0 00000080:00200000:3.0:1787827173.603538:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421894 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.603561:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728077824, len 1024, copied 1024 00000080:00200000:3.0:1787827173.603570:0:27439:0:(rw26.c:933:ll_write_end()) page@0000000054a8501e[3 000000005ce0a915 2 1 00000000d6bbf043 0 0 421894] 00000080:00200000:3.0:1787827173.603575:0:27439:0:(rw26.c:933:ll_write_end()) vmpage @00000000fb0644b7 fffffc0001001 3:0 ffff9e34c3cb2cb0 421894 lru 00000080:00200000:3.0:1787827173.603582:0:27439:0:(rw26.c:933:ll_write_end()) osc-page@00000000ff9da4bc 210950: 1< 0 - - > 2< 864051200 0 0 0x0 0x0 | 00000000640821b9 0000000006873caa > 3< 0 0 > 4< 0 0 8 484589568 - | - - + - > 5< - - + - | 0 - | 23 - -> 00000080:00200000:3.0:1787827173.603586:0:27439:0:(rw26.c:933:ll_write_end()) end page@0000000054a8501e 00000080:00200000:3.0:1787827173.603588:0:27439:0:(rw26.c:933:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1787827173.603592:0:27439:0:(vvp_io.c:1214:vvp_io_write_commit()) commit async pages: 1, from 0, to 1024 prio 0 00000008:00010000:3.0:1787827173.603602:0:27439:0:(osc_cache.c:565:osc_extent_merge()) ### extent: 00000000e070090a ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.603616:0:27439:0:(osc_cache.c:497:__osc_extent_remove()) ### extent: 00000000e070090a ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.603628:0:27439:0:(osc_cache.c:582:osc_extent_merge()) ### extent: 000000001063fa34 ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.603642:0:27439:0:(osc_cache.c:846:osc_extent_find()) ### extent: 000000001063fa34 ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.603850:0:27439:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff9e35d0e392f8, count 1, dirtied 1 00000080:00200000:3.0:1787827173.603856:0:27439:0:(vvp_io.c:1237:vvp_io_write_commit()) Committed 1 pages 1024 bytes, tot: 1024 00000080:00200000:3.0:1787827173.603862:0:27439:0:(vvp_io.c:1421:vvp_io_write_start()) oosfile: write bytes 0, result: 1024 00000020:00010000:3.0:1787827173.603874:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_release():245 00000020:00010000:3.0:1787827173.603879:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1787827173.603884:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c7d81d30 (ffff9e35c614ce18/1) at cl_lock_cancel():142 00010000:00010000:3.0:1787827173.603893:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.603909:0:27439:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 21/0,0 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1787827173.603940:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_fini():72 00000020:00010000:3.0:1787827173.603946:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c7d81d30 (ffff9e35e04b4e18/1) at cl_lock_fini():72 00000080:00200000:3.0:1787827173.603952:0:27439:0:(vvp_io.c:533:vvp_io_advance()) advancing 1024 bytes 00000080:00200000:3.0:1787827173.603956:0:27439:0:(file.c:2207:ll_file_io_generic()) Range unlock [421894, 421894] 00000080:00200000:3.0:1787827173.603964:0:27439:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787827173.603969:0:27439:0:(file.c:2240:ll_file_io_generic()) oosfile: 2 io complete with rc: 0, result: 1024, restart: 0 00000080:00200000:3.0:1787827173.603975:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728078848 i_blocks 3365288 00000080:00200000:3.0:1787827173.603981:0:27439:0:(file.c:2326:ll_file_io_generic()) iot: 2, result: 1024 00000080:00200008:3.0:1787827173.603993:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728078848, count: 1024 00000080:00200000:3.0:1787827173.604003:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421894 of 1024 to 1024 bytes 00000080:00200000:3.0:1787827173.604008:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728078848, len 1024, copied 1024 00000080:00200000:3.0:1787827173.604019:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728079872 i_blocks 3365288 00000080:00200000:3.0:1787827173.604027:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.604034:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728079872, count: 1024 00000080:00200000:3.0:1787827173.604044:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421894 of 2048 to 1024 bytes 00000080:00200000:3.0:1787827173.604060:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728079872, len 1024, copied 1024 00000080:00200000:3.0:1787827173.604079:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728080896 i_blocks 3365288 00000080:00200000:3.0:1787827173.604085:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.604092:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728080896, count: 1024 00000080:00200000:3.0:1787827173.604099:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421894 of 3072 to 1024 bytes 00000080:00200000:3.0:1787827173.604104:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728080896, len 1024, copied 1024 00000080:00200000:3.0:1787827173.604111:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728081920 i_blocks 3365288 00000080:00200000:3.0:1787827173.604117:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.604124:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728081920, count: 1024 00000080:00200000:3.0:1787827173.604131:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421895 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.604139:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 0, original count 1024 00000080:00200000:3.0:1787827173.604143:0:27439:0:(file.c:2090:ll_file_io_generic()) oosfile: write ppos: 1728081920, bytes: 1024 00000080:00200000:3.0:1787827173.604150:0:27439:0:(file.c:1848:ll_io_set_mirror()) oosfile: desiginated mirror: 0 00000020:00200000:3.0:1787827173.604160:0:27439:0:(cl_io.c:218:cl_io_rw_init()) header@000000007788600a[0x110, 369448, [0x200000402:0x27:0x0] hash] 00000020:00200000:3.0:1787827173.604163:0:27439:0:(cl_io.c:218:cl_io_rw_init()) io range: 2 [1728081920, 1728082944) 0 0 00000080:00200000:3.0:1787827173.604167:0:27439:0:(vvp_io.c:1889:vvp_io_init()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1787827173.604175:0:27439:0:(file.c:2170:ll_file_io_generic()) Range lock [421895, 421895] 00020000:00200000:3.0:1787827173.604179:0:27439:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 412 chunk: [1728081920, 1728082944) 1728082944, 1024 00020000:00200000:3.0:1787827173.604307:0:27439:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1787827173.604319:0:27439:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [421895, 421895] 00000008:00010000:3.0:1787827173.604329:0:27439:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1787827173.604335:0:27439:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff9e35c7d81d30, osc lock ffff9e35c2ddcc30, flags 0x0 00010000:00010000:3.0:1787827173.604348:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 20/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.604375:0:27439:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (864055296 864059391) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 20/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.604393:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 24/0,2 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.604432:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 24/0,2 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.604447:0:27439:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 23/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.604473:0:27439:0:(vvp_io.c:1339:vvp_io_write_start()) oosfile: write [1728081920, 1728082944) append 0 append lockpos 0 00000080:00200000:3.0:1787827173.604483:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421895 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.604505:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728081920, len 1024, copied 1024 00000080:00200000:3.0:1787827173.604515:0:27439:0:(rw26.c:933:ll_write_end()) page@000000004435803a[3 000000005ce0a915 2 1 00000000d6bbf043 0 0 421895] 00000080:00200000:3.0:1787827173.604521:0:27439:0:(rw26.c:933:ll_write_end()) vmpage @000000007daba039 fffffc0001001 3:0 ffff9e34c3cb21d0 421895 lru 00000080:00200000:3.0:1787827173.604531:0:27439:0:(rw26.c:933:ll_write_end()) osc-page@0000000058f212b1 210951: 1< 0 - - > 2< 864055296 0 0 0x0 0x0 | 00000000640821b9 0000000006873caa > 3< 0 0 > 4< 0 0 8 484585472 - | - - + - > 5< - - + - | 0 - | 24 - -> 00000080:00200000:3.0:1787827173.604534:0:27439:0:(rw26.c:933:ll_write_end()) end page@000000004435803a 00000080:00200000:3.0:1787827173.604536:0:27439:0:(rw26.c:933:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1787827173.604538:0:27439:0:(vvp_io.c:1214:vvp_io_write_commit()) commit async pages: 1, from 0, to 1024 prio 0 00000008:00010000:3.0:1787827173.604550:0:27439:0:(osc_cache.c:565:osc_extent_merge()) ### extent: 00000000e070090a ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.604570:0:27439:0:(osc_cache.c:497:__osc_extent_remove()) ### extent: 00000000e070090a ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.604593:0:27439:0:(osc_cache.c:582:osc_extent_merge()) ### extent: 000000001063fa34 ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.604618:0:27439:0:(osc_cache.c:846:osc_extent_find()) ### extent: 000000001063fa34 ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.604661:0:27439:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff9e35d0e392f8, count 1, dirtied 1 00000080:00200000:3.0:1787827173.604668:0:27439:0:(vvp_io.c:1237:vvp_io_write_commit()) Committed 1 pages 1024 bytes, tot: 1024 00000080:00200000:3.0:1787827173.604674:0:27439:0:(vvp_io.c:1421:vvp_io_write_start()) oosfile: write bytes 0, result: 1024 00000020:00010000:3.0:1787827173.604941:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_release():245 00000020:00010000:3.0:1787827173.604949:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1787827173.604956:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c7d81d30 (ffff9e35c614ce18/1) at cl_lock_cancel():142 00010000:00010000:3.0:1787827173.604963:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.604996:0:27439:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 21/0,0 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1787827173.605019:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_fini():72 00000020:00010000:3.0:1787827173.605026:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c7d81d30 (ffff9e35e04b4e18/1) at cl_lock_fini():72 00000080:00200000:3.0:1787827173.605035:0:27439:0:(vvp_io.c:533:vvp_io_advance()) advancing 1024 bytes 00000080:00200000:3.0:1787827173.605042:0:27439:0:(file.c:2207:ll_file_io_generic()) Range unlock [421895, 421895] 00000080:00200000:3.0:1787827173.605051:0:27439:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787827173.605060:0:27439:0:(file.c:2240:ll_file_io_generic()) oosfile: 2 io complete with rc: 0, result: 1024, restart: 0 00000080:00200000:3.0:1787827173.605082:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728082944 i_blocks 3365288 00000080:00200000:3.0:1787827173.605090:0:27439:0:(file.c:2326:ll_file_io_generic()) iot: 2, result: 1024 00000080:00200008:3.0:1787827173.605101:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728082944, count: 1024 00000080:00200000:3.0:1787827173.605109:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421895 of 1024 to 1024 bytes 00000080:00200000:3.0:1787827173.605116:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728082944, len 1024, copied 1024 00000080:00200000:3.0:1787827173.605125:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728083968 i_blocks 3365288 00000080:00200000:3.0:1787827173.605375:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.605386:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728083968, count: 1024 00000080:00200000:3.0:1787827173.605394:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421895 of 2048 to 1024 bytes 00000080:00200000:3.0:1787827173.605400:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728083968, len 1024, copied 1024 00000080:00200000:3.0:1787827173.605410:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728084992 i_blocks 3365288 00000080:00200000:3.0:1787827173.605415:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.605422:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728084992, count: 1024 00000080:00200000:3.0:1787827173.605428:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421895 of 3072 to 1024 bytes 00000080:00200000:3.0:1787827173.605433:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728084992, len 1024, copied 1024 00000080:00200000:3.0:1787827173.605446:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728086016 i_blocks 3365288 00000080:00200000:3.0:1787827173.605451:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.605460:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728086016, count: 1024 00000080:00200000:3.0:1787827173.605467:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421896 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.605471:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 0, original count 1024 00000080:00200000:3.0:1787827173.605477:0:27439:0:(file.c:2090:ll_file_io_generic()) oosfile: write ppos: 1728086016, bytes: 1024 00000080:00200000:3.0:1787827173.605483:0:27439:0:(file.c:1848:ll_io_set_mirror()) oosfile: desiginated mirror: 0 00000020:00200000:3.0:1787827173.605489:0:27439:0:(cl_io.c:218:cl_io_rw_init()) header@000000007788600a[0x110, 369449, [0x200000402:0x27:0x0] hash] 00000020:00200000:3.0:1787827173.605492:0:27439:0:(cl_io.c:218:cl_io_rw_init()) io range: 2 [1728086016, 1728087040) 0 0 00000080:00200000:3.0:1787827173.605497:0:27439:0:(vvp_io.c:1889:vvp_io_init()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1787827173.605507:0:27439:0:(file.c:2170:ll_file_io_generic()) Range lock [421896, 421896] 00020000:00200000:3.0:1787827173.605514:0:27439:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 412 chunk: [1728086016, 1728087040) 1728087040, 1024 00020000:00200000:3.0:1787827173.605519:0:27439:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1787827173.605529:0:27439:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [421896, 421896] 00000008:00010000:3.0:1787827173.605538:0:27439:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1787827173.605543:0:27439:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff9e35c7d81d30, osc lock ffff9e35c2ddcc30, flags 0x0 00010000:00010000:3.0:1787827173.605554:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 20/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.605569:0:27439:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (864059392 864063487) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 20/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.606392:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 24/0,2 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.607027:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 24/0,2 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.607044:0:27439:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 23/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.607084:0:27439:0:(vvp_io.c:1339:vvp_io_write_start()) oosfile: write [1728086016, 1728087040) append 0 append lockpos 0 00000080:00200000:3.0:1787827173.607108:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421896 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.607890:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728086016, len 1024, copied 1024 00000080:00200000:3.0:1787827173.607900:0:27439:0:(rw26.c:933:ll_write_end()) page@000000004c5cc88a[3 000000005ce0a915 2 1 00000000d6bbf043 0 0 421896] 00000080:00200000:3.0:1787827173.607906:0:27439:0:(rw26.c:933:ll_write_end()) vmpage @000000002b28ed9c fffffc0001001 3:0 ffff9e34c3cb32b8 421896 lru 00000080:00200000:3.0:1787827173.607929:0:27439:0:(rw26.c:933:ll_write_end()) osc-page@00000000fe3ca4e1 210952: 1< 0 - - > 2< 864059392 0 0 0x0 0x0 | 00000000640821b9 0000000006873caa > 3< 0 0 > 4< 0 0 8 484581376 - | - - + - > 5< - - + - | 0 - | 25 - -> 00000080:00200000:3.0:1787827173.607938:0:27439:0:(rw26.c:933:ll_write_end()) end page@000000004c5cc88a 00000080:00200000:3.0:1787827173.607940:0:27439:0:(rw26.c:933:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1787827173.607946:0:27439:0:(vvp_io.c:1214:vvp_io_write_commit()) commit async pages: 1, from 0, to 1024 prio 0 00000008:00010000:3.0:1787827173.607965:0:27439:0:(osc_cache.c:565:osc_extent_merge()) ### extent: 00000000e070090a ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.607982:0:27439:0:(osc_cache.c:497:__osc_extent_remove()) ### extent: 00000000e070090a ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.607998:0:27439:0:(osc_cache.c:582:osc_extent_merge()) ### extent: 000000001063fa34 ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.608012:0:27439:0:(osc_cache.c:846:osc_extent_find()) ### extent: 000000001063fa34 ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.608281:0:27439:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff9e35d0e392f8, count 1, dirtied 1 00000080:00200000:3.0:1787827173.608288:0:27439:0:(vvp_io.c:1237:vvp_io_write_commit()) Committed 1 pages 1024 bytes, tot: 1024 00000080:00200000:3.0:1787827173.608292:0:27439:0:(vvp_io.c:1421:vvp_io_write_start()) oosfile: write bytes 0, result: 1024 00000020:00010000:3.0:1787827173.608302:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_release():245 00000020:00010000:3.0:1787827173.608310:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1787827173.608314:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c7d81d30 (ffff9e35c614ce18/1) at cl_lock_cancel():142 00010000:00010000:3.0:1787827173.608324:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.608341:0:27439:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 21/0,0 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1787827173.608359:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_fini():72 00000020:00010000:3.0:1787827173.608366:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c7d81d30 (ffff9e35e04b4e18/1) at cl_lock_fini():72 00000080:00200000:3.0:1787827173.608375:0:27439:0:(vvp_io.c:533:vvp_io_advance()) advancing 1024 bytes 00000080:00200000:3.0:1787827173.608381:0:27439:0:(file.c:2207:ll_file_io_generic()) Range unlock [421896, 421896] 00000080:00200000:3.0:1787827173.608399:0:27439:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787827173.608406:0:27439:0:(file.c:2240:ll_file_io_generic()) oosfile: 2 io complete with rc: 0, result: 1024, restart: 0 00000080:00200000:3.0:1787827173.608416:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728087040 i_blocks 3365288 00000080:00200000:3.0:1787827173.608788:0:27439:0:(file.c:2326:ll_file_io_generic()) iot: 2, result: 1024 00000080:00200008:3.0:1787827173.608830:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728087040, count: 1024 00000080:00200000:3.0:1787827173.608841:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421896 of 1024 to 1024 bytes 00000080:00200000:3.0:1787827173.608847:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728087040, len 1024, copied 1024 00000080:00200000:3.0:1787827173.608858:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728088064 i_blocks 3365288 00000080:00200000:3.0:1787827173.608865:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.608871:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728088064, count: 1024 00000080:00200000:3.0:1787827173.608877:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421896 of 2048 to 1024 bytes 00000080:00200000:3.0:1787827173.608882:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728088064, len 1024, copied 1024 00000080:00200000:3.0:1787827173.608887:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728089088 i_blocks 3365288 00000080:00200000:3.0:1787827173.608889:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.608893:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728089088, count: 1024 00000080:00200000:3.0:1787827173.608896:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421896 of 3072 to 1024 bytes 00000080:00200000:3.0:1787827173.608899:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728089088, len 1024, copied 1024 00000080:00200000:3.0:1787827173.608901:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728090112 i_blocks 3365288 00000080:00200000:3.0:1787827173.608904:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.608907:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728090112, count: 1024 00000080:00200000:3.0:1787827173.608910:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421897 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.608924:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 0, original count 1024 00000080:00200000:3.0:1787827173.608929:0:27439:0:(file.c:2090:ll_file_io_generic()) oosfile: write ppos: 1728090112, bytes: 1024 00000080:00200000:3.0:1787827173.608934:0:27439:0:(file.c:1848:ll_io_set_mirror()) oosfile: desiginated mirror: 0 00000020:00200000:3.0:1787827173.608942:0:27439:0:(cl_io.c:218:cl_io_rw_init()) header@000000007788600a[0x110, 369450, [0x200000402:0x27:0x0] hash] 00000020:00200000:3.0:1787827173.608945:0:27439:0:(cl_io.c:218:cl_io_rw_init()) io range: 2 [1728090112, 1728091136) 0 0 00000080:00200000:3.0:1787827173.608950:0:27439:0:(vvp_io.c:1889:vvp_io_init()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1787827173.608961:0:27439:0:(file.c:2170:ll_file_io_generic()) Range lock [421897, 421897] 00020000:00200000:3.0:1787827173.608967:0:27439:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 412 chunk: [1728090112, 1728091136) 1728091136, 1024 00020000:00200000:3.0:1787827173.608969:0:27439:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1787827173.608976:0:27439:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [421897, 421897] 00000008:00010000:3.0:1787827173.608985:0:27439:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1787827173.608989:0:27439:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff9e35c7d81d30, osc lock ffff9e35c2ddcc30, flags 0x0 00010000:00010000:3.0:1787827173.609002:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 20/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.609024:0:27439:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (864063488 864067583) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 20/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.609047:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 24/0,2 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.609082:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 24/0,2 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.609105:0:27439:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 23/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.609130:0:27439:0:(vvp_io.c:1339:vvp_io_write_start()) oosfile: write [1728090112, 1728091136) append 0 append lockpos 0 00000080:00200000:3.0:1787827173.609139:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421897 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.609161:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728090112, len 1024, copied 1024 00000080:00200000:3.0:1787827173.609169:0:27439:0:(rw26.c:933:ll_write_end()) page@00000000b1e9393b[3 000000005ce0a915 2 1 00000000d6bbf043 0 0 421897] 00000080:00200000:3.0:1787827173.609175:0:27439:0:(rw26.c:933:ll_write_end()) vmpage @00000000a5e5b0e9 fffffc0001001 3:0 ffff9e34c3cb3488 421897 lru 00000080:00200000:3.0:1787827173.609183:0:27439:0:(rw26.c:933:ll_write_end()) osc-page@00000000edd190fb 210953: 1< 0 - - > 2< 864063488 0 0 0x0 0x0 | 00000000640821b9 0000000006873caa > 3< 0 0 > 4< 0 0 8 484577280 - | - - + - > 5< - - + - | 0 - | 26 - -> 00000080:00200000:3.0:1787827173.609188:0:27439:0:(rw26.c:933:ll_write_end()) end page@00000000b1e9393b 00000080:00200000:3.0:1787827173.609191:0:27439:0:(rw26.c:933:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1787827173.609195:0:27439:0:(vvp_io.c:1214:vvp_io_write_commit()) commit async pages: 1, from 0, to 1024 prio 0 00000008:00010000:3.0:1787827173.609205:0:27439:0:(osc_cache.c:565:osc_extent_merge()) ### extent: 00000000e070090a ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.609218:0:27439:0:(osc_cache.c:497:__osc_extent_remove()) ### extent: 00000000e070090a ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.609231:0:27439:0:(osc_cache.c:582:osc_extent_merge()) ### extent: 000000001063fa34 ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.609245:0:27439:0:(osc_cache.c:846:osc_extent_find()) ### extent: 000000001063fa34 ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.609261:0:27439:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff9e35d0e392f8, count 1, dirtied 1 00000080:00200000:3.0:1787827173.609267:0:27439:0:(vvp_io.c:1237:vvp_io_write_commit()) Committed 1 pages 1024 bytes, tot: 1024 00000080:00200000:3.0:1787827173.609271:0:27439:0:(vvp_io.c:1421:vvp_io_write_start()) oosfile: write bytes 0, result: 1024 00000020:00010000:3.0:1787827173.609282:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_release():245 00000020:00010000:3.0:1787827173.609289:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1787827173.609293:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c7d81d30 (ffff9e35c614ce18/1) at cl_lock_cancel():142 00010000:00010000:3.0:1787827173.609300:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.609312:0:27439:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 21/0,0 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1787827173.609323:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_fini():72 00000020:00010000:3.0:1787827173.609328:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c7d81d30 (ffff9e35e04b4e18/1) at cl_lock_fini():72 00000080:00200000:3.0:1787827173.609333:0:27439:0:(vvp_io.c:533:vvp_io_advance()) advancing 1024 bytes 00000080:00200000:3.0:1787827173.609337:0:27439:0:(file.c:2207:ll_file_io_generic()) Range unlock [421897, 421897] 00000080:00200000:3.0:1787827173.609343:0:27439:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787827173.609347:0:27439:0:(file.c:2240:ll_file_io_generic()) oosfile: 2 io complete with rc: 0, result: 1024, restart: 0 00000080:00200000:3.0:1787827173.609355:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728091136 i_blocks 3365288 00000080:00200000:3.0:1787827173.609360:0:27439:0:(file.c:2326:ll_file_io_generic()) iot: 2, result: 1024 00000080:00200008:3.0:1787827173.609369:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728091136, count: 1024 00000080:00200000:3.0:1787827173.609375:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421897 of 1024 to 1024 bytes 00000080:00200000:3.0:1787827173.609380:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728091136, len 1024, copied 1024 00000080:00200000:3.0:1787827173.609388:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728092160 i_blocks 3365288 00000080:00200000:3.0:1787827173.609393:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.609398:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728092160, count: 1024 00000080:00200000:3.0:1787827173.609403:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421897 of 2048 to 1024 bytes 00000080:00200000:3.0:1787827173.609408:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728092160, len 1024, copied 1024 00000080:00200000:3.0:1787827173.609413:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728093184 i_blocks 3365288 00000080:00200000:3.0:1787827173.609417:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.609422:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728093184, count: 1024 00000080:00200000:3.0:1787827173.609434:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421897 of 3072 to 1024 bytes 00000080:00200000:3.0:1787827173.609438:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728093184, len 1024, copied 1024 00000080:00200000:3.0:1787827173.609441:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728094208 i_blocks 3365288 00000080:00200000:3.0:1787827173.609444:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.609448:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728094208, count: 1024 00000080:00200000:3.0:1787827173.609454:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421898 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.609458:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 0, original count 1024 00000080:00200000:3.0:1787827173.609462:0:27439:0:(file.c:2090:ll_file_io_generic()) oosfile: write ppos: 1728094208, bytes: 1024 00000080:00200000:3.0:1787827173.609467:0:27439:0:(file.c:1848:ll_io_set_mirror()) oosfile: desiginated mirror: 0 00000020:00200000:3.0:1787827173.609475:0:27439:0:(cl_io.c:218:cl_io_rw_init()) header@000000007788600a[0x110, 369451, [0x200000402:0x27:0x0] hash] 00000020:00200000:3.0:1787827173.609479:0:27439:0:(cl_io.c:218:cl_io_rw_init()) io range: 2 [1728094208, 1728095232) 0 0 00000080:00200000:3.0:1787827173.609484:0:27439:0:(vvp_io.c:1889:vvp_io_init()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1787827173.609493:0:27439:0:(file.c:2170:ll_file_io_generic()) Range lock [421898, 421898] 00020000:00200000:3.0:1787827173.609498:0:27439:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 412 chunk: [1728094208, 1728095232) 1728095232, 1024 00020000:00200000:3.0:1787827173.609501:0:27439:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1787827173.609506:0:27439:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [421898, 421898] 00000008:00010000:3.0:1787827173.609515:0:27439:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1787827173.609520:0:27439:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff9e35c7d81d30, osc lock ffff9e35c2ddcc30, flags 0x0 00010000:00010000:3.0:1787827173.609531:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 20/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.609549:0:27439:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (864067584 864071679) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 20/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.609569:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 24/0,2 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.609591:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 24/0,2 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.609608:0:27439:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 23/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.609623:0:27439:0:(vvp_io.c:1339:vvp_io_write_start()) oosfile: write [1728094208, 1728095232) append 0 append lockpos 0 00000080:00200000:3.0:1787827173.609629:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421898 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.609649:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728094208, len 1024, copied 1024 00000080:00200000:3.0:1787827173.609657:0:27439:0:(rw26.c:933:ll_write_end()) page@000000003801008f[3 000000005ce0a915 2 1 00000000d6bbf043 0 0 421898] 00000080:00200000:3.0:1787827173.609663:0:27439:0:(rw26.c:933:ll_write_end()) vmpage @000000000f43bbef fffffc0001001 3:0 ffff9e34c3cb3bc8 421898 lru 00000080:00200000:3.0:1787827173.609675:0:27439:0:(rw26.c:933:ll_write_end()) osc-page@00000000df464413 210954: 1< 0 - - > 2< 864067584 0 0 0x0 0x0 | 00000000640821b9 0000000006873caa > 3< 0 0 > 4< 0 0 8 484573184 - | - - + - > 5< - - + - | 0 - | 27 - -> 00000080:00200000:3.0:1787827173.609679:0:27439:0:(rw26.c:933:ll_write_end()) end page@000000003801008f 00000080:00200000:3.0:1787827173.609682:0:27439:0:(rw26.c:933:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1787827173.609684:0:27439:0:(vvp_io.c:1214:vvp_io_write_commit()) commit async pages: 1, from 0, to 1024 prio 0 00000008:00010000:3.0:1787827173.609694:0:27439:0:(osc_cache.c:565:osc_extent_merge()) ### extent: 00000000e070090a ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.609708:0:27439:0:(osc_cache.c:497:__osc_extent_remove()) ### extent: 00000000e070090a ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.609719:0:27439:0:(osc_cache.c:582:osc_extent_merge()) ### extent: 000000001063fa34 ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.609731:0:27439:0:(osc_cache.c:846:osc_extent_find()) ### extent: 000000001063fa34 ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.609748:0:27439:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff9e35d0e392f8, count 1, dirtied 1 00000080:00200000:3.0:1787827173.609753:0:27439:0:(vvp_io.c:1237:vvp_io_write_commit()) Committed 1 pages 1024 bytes, tot: 1024 00000080:00200000:3.0:1787827173.609758:0:27439:0:(vvp_io.c:1421:vvp_io_write_start()) oosfile: write bytes 0, result: 1024 00000020:00010000:3.0:1787827173.609765:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_release():245 00000020:00010000:3.0:1787827173.609770:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1787827173.609774:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c7d81d30 (ffff9e35c614ce18/1) at cl_lock_cancel():142 00010000:00010000:3.0:1787827173.609781:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.609795:0:27439:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 21/0,0 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1787827173.609836:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_fini():72 00000020:00010000:3.0:1787827173.609843:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c7d81d30 (ffff9e35e04b4e18/1) at cl_lock_fini():72 00000080:00200000:3.0:1787827173.609852:0:27439:0:(vvp_io.c:533:vvp_io_advance()) advancing 1024 bytes 00000080:00200000:3.0:1787827173.609857:0:27439:0:(file.c:2207:ll_file_io_generic()) Range unlock [421898, 421898] 00000080:00200000:3.0:1787827173.609863:0:27439:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787827173.609868:0:27439:0:(file.c:2240:ll_file_io_generic()) oosfile: 2 io complete with rc: 0, result: 1024, restart: 0 00000080:00200000:3.0:1787827173.609876:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728095232 i_blocks 3365288 00000080:00200000:3.0:1787827173.609881:0:27439:0:(file.c:2326:ll_file_io_generic()) iot: 2, result: 1024 00000080:00200008:3.0:1787827173.609891:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728095232, count: 1024 00000080:00200000:3.0:1787827173.609900:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421898 of 1024 to 1024 bytes 00000080:00200000:3.0:1787827173.609905:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728095232, len 1024, copied 1024 00000080:00200000:3.0:1787827173.609913:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728096256 i_blocks 3365288 00000080:00200000:3.0:1787827173.609933:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.609940:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728096256, count: 1024 00000080:00200000:3.0:1787827173.609948:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421898 of 2048 to 1024 bytes 00000080:00200000:3.0:1787827173.609954:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728096256, len 1024, copied 1024 00000080:00200000:3.0:1787827173.609962:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728097280 i_blocks 3365288 00000080:00200000:3.0:1787827173.609968:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.609975:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728097280, count: 1024 00000080:00200000:3.0:1787827173.609982:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421898 of 3072 to 1024 bytes 00000080:00200000:3.0:1787827173.609987:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728097280, len 1024, copied 1024 00000080:00200000:3.0:1787827173.609993:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728098304 i_blocks 3365288 00000080:00200000:3.0:1787827173.610000:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.610007:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728098304, count: 1024 00000080:00200000:3.0:1787827173.610013:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421899 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.610018:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 0, original count 1024 00000080:00200000:3.0:1787827173.610023:0:27439:0:(file.c:2090:ll_file_io_generic()) oosfile: write ppos: 1728098304, bytes: 1024 00000080:00200000:3.0:1787827173.610026:0:27439:0:(file.c:1848:ll_io_set_mirror()) oosfile: desiginated mirror: 0 00000020:00200000:3.0:1787827173.610029:0:27439:0:(cl_io.c:218:cl_io_rw_init()) header@000000007788600a[0x110, 369452, [0x200000402:0x27:0x0] hash] 00000020:00200000:3.0:1787827173.610031:0:27439:0:(cl_io.c:218:cl_io_rw_init()) io range: 2 [1728098304, 1728099328) 0 0 00000080:00200000:3.0:1787827173.610035:0:27439:0:(vvp_io.c:1889:vvp_io_init()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1787827173.610042:0:27439:0:(file.c:2170:ll_file_io_generic()) Range lock [421899, 421899] 00020000:00200000:3.0:1787827173.610048:0:27439:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 412 chunk: [1728098304, 1728099328) 1728099328, 1024 00020000:00200000:3.0:1787827173.610055:0:27439:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1787827173.610063:0:27439:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [421899, 421899] 00000008:00010000:3.0:1787827173.610072:0:27439:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1787827173.610076:0:27439:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff9e35c7d81d30, osc lock ffff9e35c2ddcc30, flags 0x0 00010000:00010000:3.0:1787827173.610085:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 20/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.610100:0:27439:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (864071680 864075775) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 20/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.610117:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 24/0,2 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.610148:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 24/0,2 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.610159:0:27439:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 23/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.610173:0:27439:0:(vvp_io.c:1339:vvp_io_write_start()) oosfile: write [1728098304, 1728099328) append 0 append lockpos 0 00000080:00200000:3.0:1787827173.610178:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421899 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.610200:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728098304, len 1024, copied 1024 00000080:00200000:3.0:1787827173.610209:0:27439:0:(rw26.c:933:ll_write_end()) page@000000002ac80b6b[3 000000005ce0a915 2 1 00000000d6bbf043 0 0 421899] 00000080:00200000:3.0:1787827173.610213:0:27439:0:(rw26.c:933:ll_write_end()) vmpage @00000000ffa33b55 fffffc0001001 3:0 ffff9e34c3cb39f8 421899 lru 00000080:00200000:3.0:1787827173.610225:0:27439:0:(rw26.c:933:ll_write_end()) osc-page@00000000f47794dc 210955: 1< 0 - - > 2< 864071680 0 0 0x0 0x0 | 00000000640821b9 0000000006873caa > 3< 0 0 > 4< 0 0 8 484569088 - | - - + - > 5< - - + - | 0 - | 28 - -> 00000080:00200000:3.0:1787827173.610230:0:27439:0:(rw26.c:933:ll_write_end()) end page@000000002ac80b6b 00000080:00200000:3.0:1787827173.610232:0:27439:0:(rw26.c:933:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1787827173.610234:0:27439:0:(vvp_io.c:1214:vvp_io_write_commit()) commit async pages: 1, from 0, to 1024 prio 0 00000008:00010000:3.0:1787827173.610244:0:27439:0:(osc_cache.c:565:osc_extent_merge()) ### extent: 00000000e070090a ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.610255:0:27439:0:(osc_cache.c:497:__osc_extent_remove()) ### extent: 00000000e070090a ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.610265:0:27439:0:(osc_cache.c:582:osc_extent_merge()) ### extent: 000000001063fa34 ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.610278:0:27439:0:(osc_cache.c:846:osc_extent_find()) ### extent: 000000001063fa34 ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.610294:0:27439:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff9e35d0e392f8, count 1, dirtied 1 00000080:00200000:3.0:1787827173.610299:0:27439:0:(vvp_io.c:1237:vvp_io_write_commit()) Committed 1 pages 1024 bytes, tot: 1024 00000080:00200000:3.0:1787827173.610303:0:27439:0:(vvp_io.c:1421:vvp_io_write_start()) oosfile: write bytes 0, result: 1024 00000020:00010000:3.0:1787827173.610312:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_release():245 00000020:00010000:3.0:1787827173.610318:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1787827173.610323:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c7d81d30 (ffff9e35c614ce18/1) at cl_lock_cancel():142 00010000:00010000:3.0:1787827173.610329:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.610343:0:27439:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 21/0,0 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1787827173.610357:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_fini():72 00000020:00010000:3.0:1787827173.610363:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c7d81d30 (ffff9e35e04b4e18/1) at cl_lock_fini():72 00000080:00200000:3.0:1787827173.610369:0:27439:0:(vvp_io.c:533:vvp_io_advance()) advancing 1024 bytes 00000080:00200000:3.0:1787827173.610374:0:27439:0:(file.c:2207:ll_file_io_generic()) Range unlock [421899, 421899] 00000080:00200000:3.0:1787827173.610381:0:27439:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787827173.610388:0:27439:0:(file.c:2240:ll_file_io_generic()) oosfile: 2 io complete with rc: 0, result: 1024, restart: 0 00000080:00200000:3.0:1787827173.610396:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728099328 i_blocks 3365288 00000080:00200000:3.0:1787827173.610403:0:27439:0:(file.c:2326:ll_file_io_generic()) iot: 2, result: 1024 00000080:00200008:3.0:1787827173.610412:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728099328, count: 1024 00000080:00200000:3.0:1787827173.610420:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421899 of 1024 to 1024 bytes 00000080:00200000:3.0:1787827173.610425:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728099328, len 1024, copied 1024 00000080:00200000:3.0:1787827173.610433:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728100352 i_blocks 3365288 00000080:00200000:3.0:1787827173.610437:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.610443:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728100352, count: 1024 00000080:00200000:3.0:1787827173.610448:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421899 of 2048 to 1024 bytes 00000080:00200000:3.0:1787827173.610451:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728100352, len 1024, copied 1024 00000080:00200000:3.0:1787827173.610457:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728101376 i_blocks 3365288 00000080:00200000:3.0:1787827173.610462:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.610468:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728101376, count: 1024 00000080:00200000:3.0:1787827173.610473:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421899 of 3072 to 1024 bytes 00000080:00200000:3.0:1787827173.610477:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728101376, len 1024, copied 1024 00000080:00200000:3.0:1787827173.610483:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728102400 i_blocks 3365288 00000080:00200000:3.0:1787827173.610488:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.610493:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728102400, count: 1024 00000080:00200000:3.0:1787827173.610497:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421900 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.610500:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 0, original count 1024 00000080:00200000:3.0:1787827173.610505:0:27439:0:(file.c:2090:ll_file_io_generic()) oosfile: write ppos: 1728102400, bytes: 1024 00000080:00200000:3.0:1787827173.610509:0:27439:0:(file.c:1848:ll_io_set_mirror()) oosfile: desiginated mirror: 0 00000020:00200000:3.0:1787827173.610515:0:27439:0:(cl_io.c:218:cl_io_rw_init()) header@000000007788600a[0x110, 369453, [0x200000402:0x27:0x0] hash] 00000020:00200000:3.0:1787827173.610518:0:27439:0:(cl_io.c:218:cl_io_rw_init()) io range: 2 [1728102400, 1728103424) 0 0 00000080:00200000:3.0:1787827173.610524:0:27439:0:(vvp_io.c:1889:vvp_io_init()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1787827173.610533:0:27439:0:(file.c:2170:ll_file_io_generic()) Range lock [421900, 421900] 00020000:00200000:3.0:1787827173.610539:0:27439:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 412 chunk: [1728102400, 1728103424) 1728103424, 1024 00020000:00200000:3.0:1787827173.610544:0:27439:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1787827173.610551:0:27439:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [421900, 421900] 00000008:00010000:3.0:1787827173.610558:0:27439:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1787827173.610562:0:27439:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff9e35c7d81d30, osc lock ffff9e35c2ddcc30, flags 0x0 00010000:00010000:3.0:1787827173.610570:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 20/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.610579:0:27439:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (864075776 864079871) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 20/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.610597:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 24/0,2 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.610628:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 24/0,2 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.610637:0:27439:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 23/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.610652:0:27439:0:(vvp_io.c:1339:vvp_io_write_start()) oosfile: write [1728102400, 1728103424) append 0 append lockpos 0 00000080:00200000:3.0:1787827173.610660:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421900 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.610678:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728102400, len 1024, copied 1024 00000080:00200000:3.0:1787827173.610684:0:27439:0:(rw26.c:933:ll_write_end()) page@000000005bdf59d9[3 000000005ce0a915 2 1 00000000d6bbf043 0 0 421900] 00000080:00200000:3.0:1787827173.610690:0:27439:0:(rw26.c:933:ll_write_end()) vmpage @00000000e1548d6b fffffc0001001 3:0 ffff9e34c3cb3ae0 421900 lru 00000080:00200000:3.0:1787827173.610701:0:27439:0:(rw26.c:933:ll_write_end()) osc-page@00000000c8b45807 210956: 1< 0 - - > 2< 864075776 0 0 0x0 0x0 | 00000000640821b9 0000000006873caa > 3< 0 0 > 4< 0 0 8 484564992 - | - - + - > 5< - - + - | 0 - | 29 - -> 00000080:00200000:3.0:1787827173.610704:0:27439:0:(rw26.c:933:ll_write_end()) end page@000000005bdf59d9 00000080:00200000:3.0:1787827173.610707:0:27439:0:(rw26.c:933:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1787827173.610711:0:27439:0:(vvp_io.c:1214:vvp_io_write_commit()) commit async pages: 1, from 0, to 1024 prio 0 00000008:00010000:3.0:1787827173.610722:0:27439:0:(osc_cache.c:565:osc_extent_merge()) ### extent: 00000000e070090a ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.610737:0:27439:0:(osc_cache.c:497:__osc_extent_remove()) ### extent: 00000000e070090a ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.610757:0:27439:0:(osc_cache.c:582:osc_extent_merge()) ### extent: 000000001063fa34 ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.610773:0:27439:0:(osc_cache.c:846:osc_extent_find()) ### extent: 000000001063fa34 ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.610791:0:27439:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff9e35d0e392f8, count 1, dirtied 1 00000080:00200000:3.0:1787827173.610796:0:27439:0:(vvp_io.c:1237:vvp_io_write_commit()) Committed 1 pages 1024 bytes, tot: 1024 00000080:00200000:3.0:1787827173.610818:0:27439:0:(vvp_io.c:1421:vvp_io_write_start()) oosfile: write bytes 0, result: 1024 00000020:00010000:3.0:1787827173.610824:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_release():245 00000020:00010000:3.0:1787827173.610826:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1787827173.610828:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c7d81d30 (ffff9e35c614ce18/1) at cl_lock_cancel():142 00010000:00010000:3.0:1787827173.610832:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.610840:0:27439:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 21/0,0 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1787827173.610848:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_fini():72 00000020:00010000:3.0:1787827173.610850:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c7d81d30 (ffff9e35e04b4e18/1) at cl_lock_fini():72 00000080:00200000:3.0:1787827173.610855:0:27439:0:(vvp_io.c:533:vvp_io_advance()) advancing 1024 bytes 00000080:00200000:3.0:1787827173.610858:0:27439:0:(file.c:2207:ll_file_io_generic()) Range unlock [421900, 421900] 00000080:00200000:3.0:1787827173.610861:0:27439:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787827173.610864:0:27439:0:(file.c:2240:ll_file_io_generic()) oosfile: 2 io complete with rc: 0, result: 1024, restart: 0 00000080:00200000:3.0:1787827173.610869:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728103424 i_blocks 3365288 00000080:00200000:3.0:1787827173.610871:0:27439:0:(file.c:2326:ll_file_io_generic()) iot: 2, result: 1024 00000080:00200008:3.0:1787827173.610878:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728103424, count: 1024 00000080:00200000:3.0:1787827173.610882:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421900 of 1024 to 1024 bytes 00000080:00200000:3.0:1787827173.610884:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728103424, len 1024, copied 1024 00000080:00200000:3.0:1787827173.610889:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728104448 i_blocks 3365288 00000080:00200000:3.0:1787827173.610891:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.610897:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728104448, count: 1024 00000080:00200000:3.0:1787827173.610902:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421900 of 2048 to 1024 bytes 00000080:00200000:3.0:1787827173.610906:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728104448, len 1024, copied 1024 00000080:00200000:3.0:1787827173.610911:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728105472 i_blocks 3365288 00000080:00200000:3.0:1787827173.610930:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.610936:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728105472, count: 1024 00000080:00200000:3.0:1787827173.610943:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421900 of 3072 to 1024 bytes 00000080:00200000:3.0:1787827173.610947:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728105472, len 1024, copied 1024 00000080:00200000:3.0:1787827173.610955:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728106496 i_blocks 3365288 00000080:00200000:3.0:1787827173.610959:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.610967:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728106496, count: 1024 00000080:00200000:3.0:1787827173.610973:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421901 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.610978:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 0, original count 1024 00000080:00200000:3.0:1787827173.610982:0:27439:0:(file.c:2090:ll_file_io_generic()) oosfile: write ppos: 1728106496, bytes: 1024 00000080:00200000:3.0:1787827173.610986:0:27439:0:(file.c:1848:ll_io_set_mirror()) oosfile: desiginated mirror: 0 00000020:00200000:3.0:1787827173.610993:0:27439:0:(cl_io.c:218:cl_io_rw_init()) header@000000007788600a[0x110, 369454, [0x200000402:0x27:0x0] hash] 00000020:00200000:3.0:1787827173.610996:0:27439:0:(cl_io.c:218:cl_io_rw_init()) io range: 2 [1728106496, 1728107520) 0 0 00000080:00200000:3.0:1787827173.611001:0:27439:0:(vvp_io.c:1889:vvp_io_init()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1787827173.611011:0:27439:0:(file.c:2170:ll_file_io_generic()) Range lock [421901, 421901] 00020000:00200000:3.0:1787827173.611017:0:27439:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 412 chunk: [1728106496, 1728107520) 1728107520, 1024 00020000:00200000:3.0:1787827173.611022:0:27439:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1787827173.611030:0:27439:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [421901, 421901] 00000008:00010000:3.0:1787827173.611038:0:27439:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1787827173.611041:0:27439:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff9e35c7d81d30, osc lock ffff9e35c2ddcc30, flags 0x0 00010000:00010000:3.0:1787827173.611051:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 20/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.611063:0:27439:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (864079872 864083967) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 20/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.611079:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 24/0,2 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.611111:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 24/0,2 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.611131:0:27439:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 23/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.611148:0:27439:0:(vvp_io.c:1339:vvp_io_write_start()) oosfile: write [1728106496, 1728107520) append 0 append lockpos 0 00000080:00200000:3.0:1787827173.611155:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421901 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.611175:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728106496, len 1024, copied 1024 00000080:00200000:3.0:1787827173.611180:0:27439:0:(rw26.c:933:ll_write_end()) page@000000005aa4d39e[3 000000005ce0a915 2 1 00000000d6bbf043 0 0 421901] 00000080:00200000:3.0:1787827173.611183:0:27439:0:(rw26.c:933:ll_write_end()) vmpage @00000000a726f6a7 fffffc0001001 3:0 ffff9e34c3cb3828 421901 lru 00000080:00200000:3.0:1787827173.611189:0:27439:0:(rw26.c:933:ll_write_end()) osc-page@00000000d208f485 210957: 1< 0 - - > 2< 864079872 0 0 0x0 0x0 | 00000000640821b9 0000000006873caa > 3< 0 0 > 4< 0 0 8 484560896 - | - - + - > 5< - - + - | 0 - | 30 - -> 00000080:00200000:3.0:1787827173.611191:0:27439:0:(rw26.c:933:ll_write_end()) end page@000000005aa4d39e 00000080:00200000:3.0:1787827173.611195:0:27439:0:(rw26.c:933:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1787827173.611199:0:27439:0:(vvp_io.c:1214:vvp_io_write_commit()) commit async pages: 1, from 0, to 1024 prio 0 00000008:00010000:3.0:1787827173.611212:0:27439:0:(osc_cache.c:565:osc_extent_merge()) ### extent: 00000000e070090a ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.611225:0:27439:0:(osc_cache.c:497:__osc_extent_remove()) ### extent: 00000000e070090a ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.611246:0:27439:0:(osc_cache.c:582:osc_extent_merge()) ### extent: 000000001063fa34 ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.611265:0:27439:0:(osc_cache.c:846:osc_extent_find()) ### extent: 000000001063fa34 ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.611289:0:27439:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff9e35d0e392f8, count 1, dirtied 1 00000080:00200000:3.0:1787827173.611294:0:27439:0:(vvp_io.c:1237:vvp_io_write_commit()) Committed 1 pages 1024 bytes, tot: 1024 00000080:00200000:3.0:1787827173.611297:0:27439:0:(vvp_io.c:1421:vvp_io_write_start()) oosfile: write bytes 0, result: 1024 00000020:00010000:3.0:1787827173.611304:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_release():245 00000020:00010000:3.0:1787827173.611310:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1787827173.611313:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c7d81d30 (ffff9e35c614ce18/1) at cl_lock_cancel():142 00010000:00010000:3.0:1787827173.611323:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.611349:0:27439:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 21/0,0 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1787827173.611367:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_fini():72 00000020:00010000:3.0:1787827173.611373:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c7d81d30 (ffff9e35e04b4e18/1) at cl_lock_fini():72 00000080:00200000:3.0:1787827173.611381:0:27439:0:(vvp_io.c:533:vvp_io_advance()) advancing 1024 bytes 00000080:00200000:3.0:1787827173.611386:0:27439:0:(file.c:2207:ll_file_io_generic()) Range unlock [421901, 421901] 00000080:00200000:3.0:1787827173.611392:0:27439:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787827173.611398:0:27439:0:(file.c:2240:ll_file_io_generic()) oosfile: 2 io complete with rc: 0, result: 1024, restart: 0 00000080:00200000:3.0:1787827173.611405:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728107520 i_blocks 3365288 00000080:00200000:3.0:1787827173.611409:0:27439:0:(file.c:2326:ll_file_io_generic()) iot: 2, result: 1024 00000080:00200008:3.0:1787827173.611418:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728107520, count: 1024 00000080:00200000:3.0:1787827173.611425:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421901 of 1024 to 1024 bytes 00000080:00200000:3.0:1787827173.611430:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728107520, len 1024, copied 1024 00000080:00200000:3.0:1787827173.611440:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728108544 i_blocks 3365288 00000080:00200000:3.0:1787827173.611444:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.611452:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728108544, count: 1024 00000080:00200000:3.0:1787827173.611459:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421901 of 2048 to 1024 bytes 00000080:00200000:3.0:1787827173.611464:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728108544, len 1024, copied 1024 00000080:00200000:3.0:1787827173.611472:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728109568 i_blocks 3365288 00000080:00200000:3.0:1787827173.611476:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.611485:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728109568, count: 1024 00000080:00200000:3.0:1787827173.611493:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421901 of 3072 to 1024 bytes 00000080:00200000:3.0:1787827173.611498:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728109568, len 1024, copied 1024 00000080:00200000:3.0:1787827173.611504:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728110592 i_blocks 3365288 00000080:00200000:3.0:1787827173.611509:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.611515:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728110592, count: 1024 00000080:00200000:3.0:1787827173.611522:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421902 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.611527:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 0, original count 1024 00000080:00200000:3.0:1787827173.611533:0:27439:0:(file.c:2090:ll_file_io_generic()) oosfile: write ppos: 1728110592, bytes: 1024 00000080:00200000:3.0:1787827173.611539:0:27439:0:(file.c:1848:ll_io_set_mirror()) oosfile: desiginated mirror: 0 00000020:00200000:3.0:1787827173.611547:0:27439:0:(cl_io.c:218:cl_io_rw_init()) header@000000007788600a[0x110, 369455, [0x200000402:0x27:0x0] hash] 00000020:00200000:3.0:1787827173.611551:0:27439:0:(cl_io.c:218:cl_io_rw_init()) io range: 2 [1728110592, 1728111616) 0 0 00000080:00200000:3.0:1787827173.611556:0:27439:0:(vvp_io.c:1889:vvp_io_init()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1787827173.611566:0:27439:0:(file.c:2170:ll_file_io_generic()) Range lock [421902, 421902] 00020000:00200000:3.0:1787827173.611571:0:27439:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 412 chunk: [1728110592, 1728111616) 1728111616, 1024 00020000:00200000:3.0:1787827173.611578:0:27439:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1787827173.611585:0:27439:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [421902, 421902] 00000008:00010000:3.0:1787827173.611593:0:27439:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1787827173.611598:0:27439:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff9e35c7d81d30, osc lock ffff9e35c2ddcc30, flags 0x0 00010000:00010000:3.0:1787827173.611610:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 20/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.611633:0:27439:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (864083968 864088063) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 20/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.611649:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 24/0,2 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.611666:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 24/0,2 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.611688:0:27439:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 23/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.611700:0:27439:0:(vvp_io.c:1339:vvp_io_write_start()) oosfile: write [1728110592, 1728111616) append 0 append lockpos 0 00000080:00200000:3.0:1787827173.611709:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421902 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.611736:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728110592, len 1024, copied 1024 00000080:00200000:3.0:1787827173.611745:0:27439:0:(rw26.c:933:ll_write_end()) page@00000000b7833bf4[3 000000005ce0a915 2 1 00000000d6bbf043 0 0 421902] 00000080:00200000:3.0:1787827173.611752:0:27439:0:(rw26.c:933:ll_write_end()) vmpage @00000000b7334581 fffffc0001021 2:0 ffff9e34c3cb3570 421902 lru 00000080:00200000:3.0:1787827173.611760:0:27439:0:(rw26.c:933:ll_write_end()) osc-page@0000000048c619d8 210958: 1< 0 - - > 2< 864083968 0 0 0x0 0x0 | 00000000640821b9 0000000006873caa > 3< 0 0 > 4< 0 0 8 484556800 - | - - + - > 5< - - + - | 0 - | 31 - -> 00000080:00200000:3.0:1787827173.611764:0:27439:0:(rw26.c:933:ll_write_end()) end page@00000000b7833bf4 00000080:00200000:3.0:1787827173.611768:0:27439:0:(rw26.c:933:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1787827173.611772:0:27439:0:(vvp_io.c:1214:vvp_io_write_commit()) commit async pages: 1, from 0, to 1024 prio 0 00000008:00010000:3.0:1787827173.611782:0:27439:0:(osc_cache.c:565:osc_extent_merge()) ### extent: 00000000e070090a ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.612476:0:27439:0:(osc_cache.c:497:__osc_extent_remove()) ### extent: 00000000e070090a ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.612494:0:27439:0:(osc_cache.c:582:osc_extent_merge()) ### extent: 000000001063fa34 ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.612513:0:27439:0:(osc_cache.c:846:osc_extent_find()) ### extent: 000000001063fa34 ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.612534:0:27439:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff9e35d0e392f8, count 1, dirtied 1 00000080:00200000:3.0:1787827173.612540:0:27439:0:(vvp_io.c:1237:vvp_io_write_commit()) Committed 1 pages 1024 bytes, tot: 1024 00000080:00200000:3.0:1787827173.612545:0:27439:0:(vvp_io.c:1421:vvp_io_write_start()) oosfile: write bytes 0, result: 1024 00000020:00010000:3.0:1787827173.612557:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_release():245 00000020:00010000:3.0:1787827173.612562:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1787827173.612568:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c7d81d30 (ffff9e35c614ce18/1) at cl_lock_cancel():142 00010000:00010000:3.0:1787827173.612575:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.612591:0:27439:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 21/0,0 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1787827173.612610:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_fini():72 00000020:00010000:3.0:1787827173.613098:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c7d81d30 (ffff9e35e04b4e18/1) at cl_lock_fini():72 00000080:00200000:3.0:1787827173.613106:0:27439:0:(vvp_io.c:533:vvp_io_advance()) advancing 1024 bytes 00000080:00200000:3.0:1787827173.613113:0:27439:0:(file.c:2207:ll_file_io_generic()) Range unlock [421902, 421902] 00000080:00200000:3.0:1787827173.613120:0:27439:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787827173.613130:0:27439:0:(file.c:2240:ll_file_io_generic()) oosfile: 2 io complete with rc: 0, result: 1024, restart: 0 00000080:00200000:3.0:1787827173.613138:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728111616 i_blocks 3365288 00000080:00200000:3.0:1787827173.613146:0:27439:0:(file.c:2326:ll_file_io_generic()) iot: 2, result: 1024 00000080:00200008:3.0:1787827173.613159:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728111616, count: 1024 00000080:00200000:3.0:1787827173.613170:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421902 of 1024 to 1024 bytes 00000080:00200000:3.0:1787827173.613175:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728111616, len 1024, copied 1024 00000080:00200000:3.0:1787827173.613183:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728112640 i_blocks 3365288 00000080:00200000:3.0:1787827173.613185:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.613189:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728112640, count: 1024 00000080:00200000:3.0:1787827173.613194:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421902 of 2048 to 1024 bytes 00000080:00200000:3.0:1787827173.613200:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728112640, len 1024, copied 1024 00000080:00200000:3.0:1787827173.613205:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728113664 i_blocks 3365288 00000080:00200000:3.0:1787827173.613210:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.613215:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728113664, count: 1024 00000080:00200000:3.0:1787827173.613219:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421902 of 3072 to 1024 bytes 00000080:00200000:3.0:1787827173.613222:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728113664, len 1024, copied 1024 00000080:00200000:3.0:1787827173.613225:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728114688 i_blocks 3365288 00000080:00200000:3.0:1787827173.613227:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.613231:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728114688, count: 1024 00000080:00200000:3.0:1787827173.613236:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421903 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.613240:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 0, original count 1024 00000080:00200000:3.0:1787827173.613242:0:27439:0:(file.c:2090:ll_file_io_generic()) oosfile: write ppos: 1728114688, bytes: 1024 00000080:00200000:3.0:1787827173.613246:0:27439:0:(file.c:1848:ll_io_set_mirror()) oosfile: desiginated mirror: 0 00000020:00200000:3.0:1787827173.613251:0:27439:0:(cl_io.c:218:cl_io_rw_init()) header@000000007788600a[0x110, 369456, [0x200000402:0x27:0x0] hash] 00000020:00200000:3.0:1787827173.613253:0:27439:0:(cl_io.c:218:cl_io_rw_init()) io range: 2 [1728114688, 1728115712) 0 0 00000080:00200000:3.0:1787827173.613258:0:27439:0:(vvp_io.c:1889:vvp_io_init()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1787827173.613267:0:27439:0:(file.c:2170:ll_file_io_generic()) Range lock [421903, 421903] 00020000:00200000:3.0:1787827173.613270:0:27439:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 412 chunk: [1728114688, 1728115712) 1728115712, 1024 00020000:00200000:3.0:1787827173.613276:0:27439:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1787827173.613283:0:27439:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [421903, 421903] 00000008:00010000:3.0:1787827173.613289:0:27439:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1787827173.613292:0:27439:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff9e35c7d81d30, osc lock ffff9e35c2ddcc30, flags 0x0 00010000:00010000:3.0:1787827173.613301:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 20/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.613314:0:27439:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (864088064 864092159) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 20/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.613329:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 24/0,2 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.613362:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 24/0,2 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.613374:0:27439:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 23/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.613390:0:27439:0:(vvp_io.c:1339:vvp_io_write_start()) oosfile: write [1728114688, 1728115712) append 0 append lockpos 0 00000080:00200000:3.0:1787827173.613398:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421903 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.613417:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728114688, len 1024, copied 1024 00000080:00200000:3.0:1787827173.613424:0:27439:0:(rw26.c:933:ll_write_end()) page@00000000ca41fcaa[3 000000005ce0a915 2 1 00000000d6bbf043 0 0 421903] 00000080:00200000:3.0:1787827173.613429:0:27439:0:(rw26.c:933:ll_write_end()) vmpage @0000000048af984c fffffc0001001 3:0 ffff9e34c3cb3e80 421903 lru 00000080:00200000:3.0:1787827173.613436:0:27439:0:(rw26.c:933:ll_write_end()) osc-page@0000000066c33350 210959: 1< 0 - - > 2< 864088064 0 0 0x0 0x0 | 00000000640821b9 0000000006873caa > 3< 0 0 > 4< 0 0 8 484552704 - | - - + - > 5< - - + - | 0 - | 32 - -> 00000080:00200000:3.0:1787827173.613439:0:27439:0:(rw26.c:933:ll_write_end()) end page@00000000ca41fcaa 00000080:00200000:3.0:1787827173.613441:0:27439:0:(rw26.c:933:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1787827173.613444:0:27439:0:(vvp_io.c:1214:vvp_io_write_commit()) commit async pages: 1, from 0, to 1024 prio 0 00000008:00010000:3.0:1787827173.613454:0:27439:0:(osc_cache.c:565:osc_extent_merge()) ### extent: 00000000e070090a ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.613469:0:27439:0:(osc_cache.c:497:__osc_extent_remove()) ### extent: 00000000e070090a ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.613482:0:27439:0:(osc_cache.c:582:osc_extent_merge()) ### extent: 000000001063fa34 ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.613494:0:27439:0:(osc_cache.c:846:osc_extent_find()) ### extent: 000000001063fa34 ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.613511:0:27439:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff9e35d0e392f8, count 1, dirtied 1 00000080:00200000:3.0:1787827173.613515:0:27439:0:(vvp_io.c:1237:vvp_io_write_commit()) Committed 1 pages 1024 bytes, tot: 1024 00000080:00200000:3.0:1787827173.613518:0:27439:0:(vvp_io.c:1421:vvp_io_write_start()) oosfile: write bytes 0, result: 1024 00000020:00010000:3.0:1787827173.613523:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_release():245 00000020:00010000:3.0:1787827173.613529:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1787827173.613532:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c7d81d30 (ffff9e35c614ce18/1) at cl_lock_cancel():142 00010000:00010000:3.0:1787827173.613537:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.613548:0:27439:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 21/0,0 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1787827173.613561:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_fini():72 00000020:00010000:3.0:1787827173.613564:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c7d81d30 (ffff9e35e04b4e18/1) at cl_lock_fini():72 00000080:00200000:3.0:1787827173.613570:0:27439:0:(vvp_io.c:533:vvp_io_advance()) advancing 1024 bytes 00000080:00200000:3.0:1787827173.613573:0:27439:0:(file.c:2207:ll_file_io_generic()) Range unlock [421903, 421903] 00000080:00200000:3.0:1787827173.613578:0:27439:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787827173.613584:0:27439:0:(file.c:2240:ll_file_io_generic()) oosfile: 2 io complete with rc: 0, result: 1024, restart: 0 00000080:00200000:3.0:1787827173.613591:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728115712 i_blocks 3365288 00000080:00200000:3.0:1787827173.613593:0:27439:0:(file.c:2326:ll_file_io_generic()) iot: 2, result: 1024 00000080:00200008:3.0:1787827173.613602:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728115712, count: 1024 00000080:00200000:3.0:1787827173.613609:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421903 of 1024 to 1024 bytes 00000080:00200000:3.0:1787827173.613613:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728115712, len 1024, copied 1024 00000080:00200000:3.0:1787827173.613617:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728116736 i_blocks 3365288 00000080:00200000:3.0:1787827173.613619:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.613623:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728116736, count: 1024 00000080:00200000:3.0:1787827173.613628:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421903 of 2048 to 1024 bytes 00000080:00200000:3.0:1787827173.613631:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728116736, len 1024, copied 1024 00000080:00200000:3.0:1787827173.613636:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728117760 i_blocks 3365288 00000080:00200000:3.0:1787827173.613637:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.613640:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728117760, count: 1024 00000080:00200000:3.0:1787827173.613645:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421903 of 3072 to 1024 bytes 00000080:00200000:3.0:1787827173.613649:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728117760, len 1024, copied 1024 00000080:00200000:3.0:1787827173.613653:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728118784 i_blocks 3365288 00000080:00200000:3.0:1787827173.613658:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.613662:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728118784, count: 1024 00000080:00200000:3.0:1787827173.613667:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421904 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.613670:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 0, original count 1024 00000080:00200000:3.0:1787827173.613672:0:27439:0:(file.c:2090:ll_file_io_generic()) oosfile: write ppos: 1728118784, bytes: 1024 00000080:00200000:3.0:1787827173.613674:0:27439:0:(file.c:1848:ll_io_set_mirror()) oosfile: desiginated mirror: 0 00000020:00200000:3.0:1787827173.613690:0:27439:0:(cl_io.c:218:cl_io_rw_init()) header@000000007788600a[0x110, 369457, [0x200000402:0x27:0x0] hash] 00000020:00200000:3.0:1787827173.613693:0:27439:0:(cl_io.c:218:cl_io_rw_init()) io range: 2 [1728118784, 1728119808) 0 0 00000080:00200000:3.0:1787827173.613696:0:27439:0:(vvp_io.c:1889:vvp_io_init()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1787827173.613705:0:27439:0:(file.c:2170:ll_file_io_generic()) Range lock [421904, 421904] 00020000:00200000:3.0:1787827173.613711:0:27439:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 412 chunk: [1728118784, 1728119808) 1728119808, 1024 00020000:00200000:3.0:1787827173.613718:0:27439:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1787827173.613725:0:27439:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [421904, 421904] 00000008:00010000:3.0:1787827173.613733:0:27439:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1787827173.613737:0:27439:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff9e35c7d81d30, osc lock ffff9e35c2ddcc30, flags 0x0 00010000:00010000:3.0:1787827173.613747:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 20/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.613763:0:27439:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (864092160 864096255) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 20/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.613784:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 24/0,2 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.613826:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 24/0,2 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.613845:0:27439:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 23/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.613871:0:27439:0:(vvp_io.c:1339:vvp_io_write_start()) oosfile: write [1728118784, 1728119808) append 0 append lockpos 0 00000080:00200000:3.0:1787827173.613881:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421904 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.613902:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728118784, len 1024, copied 1024 00000080:00200000:3.0:1787827173.613911:0:27439:0:(rw26.c:933:ll_write_end()) page@00000000d2309391[3 000000005ce0a915 2 1 00000000d6bbf043 0 0 421904] 00000080:00200000:3.0:1787827173.613930:0:27439:0:(rw26.c:933:ll_write_end()) vmpage @000000007a696b9b fffffc0001001 3:0 ffff9e34c3cb3740 421904 lru 00000080:00200000:3.0:1787827173.613944:0:27439:0:(rw26.c:933:ll_write_end()) osc-page@0000000010919093 210960: 1< 0 - - > 2< 864092160 0 0 0x0 0x0 | 00000000640821b9 0000000006873caa > 3< 0 0 > 4< 0 0 8 484548608 - | - - + - > 5< - - + - | 0 - | 33 - -> 00000080:00200000:3.0:1787827173.613949:0:27439:0:(rw26.c:933:ll_write_end()) end page@00000000d2309391 00000080:00200000:3.0:1787827173.613950:0:27439:0:(rw26.c:933:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1787827173.613954:0:27439:0:(vvp_io.c:1214:vvp_io_write_commit()) commit async pages: 1, from 0, to 1024 prio 0 00000008:00010000:3.0:1787827173.613969:0:27439:0:(osc_cache.c:565:osc_extent_merge()) ### extent: 00000000e070090a ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.613984:0:27439:0:(osc_cache.c:497:__osc_extent_remove()) ### extent: 00000000e070090a ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.613998:0:27439:0:(osc_cache.c:582:osc_extent_merge()) ### extent: 000000001063fa34 ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.614011:0:27439:0:(osc_cache.c:846:osc_extent_find()) ### extent: 000000001063fa34 ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.614031:0:27439:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff9e35d0e392f8, count 1, dirtied 1 00000080:00200000:3.0:1787827173.614036:0:27439:0:(vvp_io.c:1237:vvp_io_write_commit()) Committed 1 pages 1024 bytes, tot: 1024 00000080:00200000:3.0:1787827173.614039:0:27439:0:(vvp_io.c:1421:vvp_io_write_start()) oosfile: write bytes 0, result: 1024 00000020:00010000:3.0:1787827173.614050:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_release():245 00000020:00010000:3.0:1787827173.614056:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1787827173.614062:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c7d81d30 (ffff9e35c614ce18/1) at cl_lock_cancel():142 00010000:00010000:3.0:1787827173.614071:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.614086:0:27439:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 21/0,0 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1787827173.614103:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_fini():72 00000020:00010000:3.0:1787827173.614108:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c7d81d30 (ffff9e35e04b4e18/1) at cl_lock_fini():72 00000080:00200000:3.0:1787827173.614117:0:27439:0:(vvp_io.c:533:vvp_io_advance()) advancing 1024 bytes 00000080:00200000:3.0:1787827173.614122:0:27439:0:(file.c:2207:ll_file_io_generic()) Range unlock [421904, 421904] 00000080:00200000:3.0:1787827173.614129:0:27439:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787827173.614136:0:27439:0:(file.c:2240:ll_file_io_generic()) oosfile: 2 io complete with rc: 0, result: 1024, restart: 0 00000080:00200000:3.0:1787827173.614143:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728119808 i_blocks 3365288 00000080:00200000:3.0:1787827173.614150:0:27439:0:(file.c:2326:ll_file_io_generic()) iot: 2, result: 1024 00000080:00200008:3.0:1787827173.614162:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728119808, count: 1024 00000080:00200000:3.0:1787827173.614170:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421904 of 1024 to 1024 bytes 00000080:00200000:3.0:1787827173.614177:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728119808, len 1024, copied 1024 00000080:00200000:3.0:1787827173.614185:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728120832 i_blocks 3365288 00000080:00200000:3.0:1787827173.614193:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.614202:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728120832, count: 1024 00000080:00200000:3.0:1787827173.614210:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421904 of 2048 to 1024 bytes 00000080:00200000:3.0:1787827173.614214:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728120832, len 1024, copied 1024 00000080:00200000:3.0:1787827173.614221:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728121856 i_blocks 3365288 00000080:00200000:3.0:1787827173.614225:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.614230:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728121856, count: 1024 00000080:00200000:3.0:1787827173.614237:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421904 of 3072 to 1024 bytes 00000080:00200000:3.0:1787827173.614241:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728121856, len 1024, copied 1024 00000080:00200000:3.0:1787827173.614247:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728122880 i_blocks 3365288 00000080:00200000:3.0:1787827173.614254:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.614260:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728122880, count: 1024 00000080:00200000:3.0:1787827173.614268:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421905 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.614273:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 0, original count 1024 00000080:00200000:3.0:1787827173.614276:0:27439:0:(file.c:2090:ll_file_io_generic()) oosfile: write ppos: 1728122880, bytes: 1024 00000080:00200000:3.0:1787827173.614279:0:27439:0:(file.c:1848:ll_io_set_mirror()) oosfile: desiginated mirror: 0 00000020:00200000:3.0:1787827173.614289:0:27439:0:(cl_io.c:218:cl_io_rw_init()) header@000000007788600a[0x110, 369458, [0x200000402:0x27:0x0] hash] 00000020:00200000:3.0:1787827173.614292:0:27439:0:(cl_io.c:218:cl_io_rw_init()) io range: 2 [1728122880, 1728123904) 0 0 00000080:00200000:3.0:1787827173.614297:0:27439:0:(vvp_io.c:1889:vvp_io_init()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1787827173.614306:0:27439:0:(file.c:2170:ll_file_io_generic()) Range lock [421905, 421905] 00020000:00200000:3.0:1787827173.614314:0:27439:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 412 chunk: [1728122880, 1728123904) 1728123904, 1024 00020000:00200000:3.0:1787827173.614320:0:27439:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1787827173.614327:0:27439:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [421905, 421905] 00000008:00010000:3.0:1787827173.614335:0:27439:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1787827173.614339:0:27439:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff9e35c7d81d30, osc lock ffff9e35c2ddcc30, flags 0x0 00010000:00010000:3.0:1787827173.614350:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 20/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.614360:0:27439:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (864096256 864100351) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 20/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.614378:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 24/0,2 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.614409:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 24/0,2 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.614419:0:27439:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 23/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.614436:0:27439:0:(vvp_io.c:1339:vvp_io_write_start()) oosfile: write [1728122880, 1728123904) append 0 append lockpos 0 00000080:00200000:3.0:1787827173.614443:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421905 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.614472:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728122880, len 1024, copied 1024 00000080:00200000:3.0:1787827173.614481:0:27439:0:(rw26.c:933:ll_write_end()) page@00000000a4237b6d[3 000000005ce0a915 2 1 00000000d6bbf043 0 0 421905] 00000080:00200000:3.0:1787827173.614487:0:27439:0:(rw26.c:933:ll_write_end()) vmpage @0000000010e5c556 fffffc0001001 3:0 ffff9e34c3cb3910 421905 lru 00000080:00200000:3.0:1787827173.614497:0:27439:0:(rw26.c:933:ll_write_end()) osc-page@00000000ef578218 210961: 1< 0 - - > 2< 864096256 0 0 0x0 0x0 | 00000000640821b9 0000000006873caa > 3< 0 0 > 4< 0 0 8 484544512 - | - - + - > 5< - - + - | 0 - | 34 - -> 00000080:00200000:3.0:1787827173.614502:0:27439:0:(rw26.c:933:ll_write_end()) end page@00000000a4237b6d 00000080:00200000:3.0:1787827173.614505:0:27439:0:(rw26.c:933:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1787827173.614510:0:27439:0:(vvp_io.c:1214:vvp_io_write_commit()) commit async pages: 1, from 0, to 1024 prio 0 00000008:00010000:3.0:1787827173.614524:0:27439:0:(osc_cache.c:565:osc_extent_merge()) ### extent: 00000000e070090a ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.614538:0:27439:0:(osc_cache.c:497:__osc_extent_remove()) ### extent: 00000000e070090a ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.614549:0:27439:0:(osc_cache.c:582:osc_extent_merge()) ### extent: 000000001063fa34 ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.614563:0:27439:0:(osc_cache.c:846:osc_extent_find()) ### extent: 000000001063fa34 ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.614582:0:27439:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff9e35d0e392f8, count 1, dirtied 1 00000080:00200000:3.0:1787827173.614589:0:27439:0:(vvp_io.c:1237:vvp_io_write_commit()) Committed 1 pages 1024 bytes, tot: 1024 00000080:00200000:3.0:1787827173.614593:0:27439:0:(vvp_io.c:1421:vvp_io_write_start()) oosfile: write bytes 0, result: 1024 00000020:00010000:3.0:1787827173.614602:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_release():245 00000020:00010000:3.0:1787827173.614608:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1787827173.614613:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c7d81d30 (ffff9e35c614ce18/1) at cl_lock_cancel():142 00010000:00010000:3.0:1787827173.614619:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.614637:0:27439:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 21/0,0 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1787827173.614656:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_fini():72 00000020:00010000:3.0:1787827173.614662:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c7d81d30 (ffff9e35e04b4e18/1) at cl_lock_fini():72 00000080:00200000:3.0:1787827173.614669:0:27439:0:(vvp_io.c:533:vvp_io_advance()) advancing 1024 bytes 00000080:00200000:3.0:1787827173.614674:0:27439:0:(file.c:2207:ll_file_io_generic()) Range unlock [421905, 421905] 00000080:00200000:3.0:1787827173.614680:0:27439:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787827173.614684:0:27439:0:(file.c:2240:ll_file_io_generic()) oosfile: 2 io complete with rc: 0, result: 1024, restart: 0 00000080:00200000:3.0:1787827173.614692:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728123904 i_blocks 3365288 00000080:00200000:3.0:1787827173.614699:0:27439:0:(file.c:2326:ll_file_io_generic()) iot: 2, result: 1024 00000080:00200008:3.0:1787827173.614710:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728123904, count: 1024 00000080:00200000:3.0:1787827173.614720:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421905 of 1024 to 1024 bytes 00000080:00200000:3.0:1787827173.614727:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728123904, len 1024, copied 1024 00000080:00200000:3.0:1787827173.614736:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728124928 i_blocks 3365288 00000080:00200000:3.0:1787827173.614743:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.614748:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728124928, count: 1024 00000080:00200000:3.0:1787827173.614754:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421905 of 2048 to 1024 bytes 00000080:00200000:3.0:1787827173.614758:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728124928, len 1024, copied 1024 00000080:00200000:3.0:1787827173.614765:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728125952 i_blocks 3365288 00000080:00200000:3.0:1787827173.614771:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.615332:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728125952, count: 1024 00000080:00200000:3.0:1787827173.615341:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421905 of 3072 to 1024 bytes 00000080:00200000:3.0:1787827173.615346:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728125952, len 1024, copied 1024 00000080:00200000:3.0:1787827173.615357:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728126976 i_blocks 3365288 00000080:00200000:3.0:1787827173.615362:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.615370:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728126976, count: 1024 00000080:00200000:3.0:1787827173.615378:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421906 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.615381:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 0, original count 1024 00000080:00200000:3.0:1787827173.615384:0:27439:0:(file.c:2090:ll_file_io_generic()) oosfile: write ppos: 1728126976, bytes: 1024 00000080:00200000:3.0:1787827173.615426:0:27439:0:(file.c:1848:ll_io_set_mirror()) oosfile: desiginated mirror: 0 00000020:00200000:3.0:1787827173.615435:0:27439:0:(cl_io.c:218:cl_io_rw_init()) header@000000007788600a[0x110, 369459, [0x200000402:0x27:0x0] hash] 00000020:00200000:3.0:1787827173.615438:0:27439:0:(cl_io.c:218:cl_io_rw_init()) io range: 2 [1728126976, 1728128000) 0 0 00000080:00200000:3.0:1787827173.615443:0:27439:0:(vvp_io.c:1889:vvp_io_init()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1787827173.615452:0:27439:0:(file.c:2170:ll_file_io_generic()) Range lock [421906, 421906] 00020000:00200000:3.0:1787827173.615459:0:27439:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 412 chunk: [1728126976, 1728128000) 1728128000, 1024 00020000:00200000:3.0:1787827173.615464:0:27439:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1787827173.615473:0:27439:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [421906, 421906] 00000008:00010000:3.0:1787827173.615481:0:27439:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1787827173.615486:0:27439:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff9e35c7d81d30, osc lock ffff9e35c2ddcc30, flags 0x0 00010000:00010000:3.0:1787827173.615497:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 20/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.615511:0:27439:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (864100352 864104447) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 20/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.615524:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 24/0,2 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.615553:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 24/0,2 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.615569:0:27439:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 23/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.615590:0:27439:0:(vvp_io.c:1339:vvp_io_write_start()) oosfile: write [1728126976, 1728128000) append 0 append lockpos 0 00000080:00200000:3.0:1787827173.615598:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421906 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.615619:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728126976, len 1024, copied 1024 00000080:00200000:3.0:1787827173.615625:0:27439:0:(rw26.c:933:ll_write_end()) page@00000000978d9a2d[3 000000005ce0a915 2 1 00000000d6bbf043 0 0 421906] 00000080:00200000:3.0:1787827173.615628:0:27439:0:(rw26.c:933:ll_write_end()) vmpage @0000000002b0de93 fffffc0001001 3:0 ffff9e34c3cb30e8 421906 lru 00000080:00200000:3.0:1787827173.615634:0:27439:0:(rw26.c:933:ll_write_end()) osc-page@00000000cf3aad0b 210962: 1< 0 - - > 2< 864100352 0 0 0x0 0x0 | 00000000640821b9 0000000006873caa > 3< 0 0 > 4< 0 0 8 484540416 - | - - + - > 5< - - + - | 0 - | 35 - -> 00000080:00200000:3.0:1787827173.615636:0:27439:0:(rw26.c:933:ll_write_end()) end page@00000000978d9a2d 00000080:00200000:3.0:1787827173.615637:0:27439:0:(rw26.c:933:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1787827173.615640:0:27439:0:(vvp_io.c:1214:vvp_io_write_commit()) commit async pages: 1, from 0, to 1024 prio 0 00000008:00010000:3.0:1787827173.615653:0:27439:0:(osc_cache.c:565:osc_extent_merge()) ### extent: 00000000e070090a ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.615673:0:27439:0:(osc_cache.c:497:__osc_extent_remove()) ### extent: 00000000e070090a ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.615687:0:27439:0:(osc_cache.c:582:osc_extent_merge()) ### extent: 000000001063fa34 ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.615695:0:27439:0:(osc_cache.c:846:osc_extent_find()) ### extent: 000000001063fa34 ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.615722:0:27439:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff9e35d0e392f8, count 1, dirtied 1 00000080:00200000:3.0:1787827173.615727:0:27439:0:(vvp_io.c:1237:vvp_io_write_commit()) Committed 1 pages 1024 bytes, tot: 1024 00000080:00200000:3.0:1787827173.615732:0:27439:0:(vvp_io.c:1421:vvp_io_write_start()) oosfile: write bytes 0, result: 1024 00000020:00010000:3.0:1787827173.615741:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_release():245 00000020:00010000:3.0:1787827173.615746:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1787827173.615753:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c7d81d30 (ffff9e35c614ce18/1) at cl_lock_cancel():142 00010000:00010000:3.0:1787827173.615762:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.615772:0:27439:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 21/0,0 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1787827173.615792:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_fini():72 00000020:00010000:3.0:1787827173.615814:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c7d81d30 (ffff9e35e04b4e18/1) at cl_lock_fini():72 00000080:00200000:3.0:1787827173.615823:0:27439:0:(vvp_io.c:533:vvp_io_advance()) advancing 1024 bytes 00000080:00200000:3.0:1787827173.615829:0:27439:0:(file.c:2207:ll_file_io_generic()) Range unlock [421906, 421906] 00000080:00200000:3.0:1787827173.615835:0:27439:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787827173.615844:0:27439:0:(file.c:2240:ll_file_io_generic()) oosfile: 2 io complete with rc: 0, result: 1024, restart: 0 00000080:00200000:3.0:1787827173.615851:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728128000 i_blocks 3365288 00000080:00200000:3.0:1787827173.615858:0:27439:0:(file.c:2326:ll_file_io_generic()) iot: 2, result: 1024 00000080:00200008:3.0:1787827173.615868:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728128000, count: 1024 00000080:00200000:3.0:1787827173.615877:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421906 of 1024 to 1024 bytes 00000080:00200000:3.0:1787827173.615882:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728128000, len 1024, copied 1024 00000080:00200000:3.0:1787827173.615890:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728129024 i_blocks 3365288 00000080:00200000:3.0:1787827173.615896:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.615903:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728129024, count: 1024 00000080:00200000:3.0:1787827173.615909:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421906 of 2048 to 1024 bytes 00000080:00200000:3.0:1787827173.615927:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728129024, len 1024, copied 1024 00000080:00200000:3.0:1787827173.615935:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728130048 i_blocks 3365288 00000080:00200000:3.0:1787827173.615940:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.615950:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728130048, count: 1024 00000080:00200000:3.0:1787827173.615957:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421906 of 3072 to 1024 bytes 00000080:00200000:3.0:1787827173.615964:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728130048, len 1024, copied 1024 00000080:00200000:3.0:1787827173.615972:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728131072 i_blocks 3365288 00000080:00200000:3.0:1787827173.615979:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.615988:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728131072, count: 1024 00000080:00200000:3.0:1787827173.615996:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421907 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.616001:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 0, original count 1024 00000080:00200000:3.0:1787827173.616005:0:27439:0:(file.c:2090:ll_file_io_generic()) oosfile: write ppos: 1728131072, bytes: 1024 00000080:00200000:3.0:1787827173.616009:0:27439:0:(file.c:1848:ll_io_set_mirror()) oosfile: desiginated mirror: 0 00000020:00200000:3.0:1787827173.616016:0:27439:0:(cl_io.c:218:cl_io_rw_init()) header@000000007788600a[0x110, 369460, [0x200000402:0x27:0x0] hash] 00000020:00200000:3.0:1787827173.616019:0:27439:0:(cl_io.c:218:cl_io_rw_init()) io range: 2 [1728131072, 1728132096) 0 0 00000080:00200000:3.0:1787827173.616023:0:27439:0:(vvp_io.c:1889:vvp_io_init()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1787827173.616033:0:27439:0:(file.c:2170:ll_file_io_generic()) Range lock [421907, 421907] 00020000:00200000:3.0:1787827173.616039:0:27439:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 412 chunk: [1728131072, 1728132096) 1728132096, 1024 00020000:00200000:3.0:1787827173.616044:0:27439:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1787827173.616053:0:27439:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [421907, 421907] 00000008:00010000:3.0:1787827173.616063:0:27439:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1787827173.616068:0:27439:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff9e35c7d81d30, osc lock ffff9e35c2ddcc30, flags 0x0 00010000:00010000:3.0:1787827173.616080:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 20/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.616098:0:27439:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (864104448 864108543) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 20/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.616115:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 24/0,2 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.616136:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 24/0,2 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.616152:0:27439:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 23/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.616175:0:27439:0:(vvp_io.c:1339:vvp_io_write_start()) oosfile: write [1728131072, 1728132096) append 0 append lockpos 0 00000080:00200000:3.0:1787827173.616182:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421907 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.616207:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728131072, len 1024, copied 1024 00000080:00200000:3.0:1787827173.616215:0:27439:0:(rw26.c:933:ll_write_end()) page@00000000a7cde2eb[3 000000005ce0a915 2 1 00000000d6bbf043 0 0 421907] 00000080:00200000:3.0:1787827173.616220:0:27439:0:(rw26.c:933:ll_write_end()) vmpage @00000000138c66ff fffffc0001001 3:0 ffff9e34c3cb3cb0 421907 lru 00000080:00200000:3.0:1787827173.616363:0:27439:0:(rw26.c:933:ll_write_end()) osc-page@00000000665f2742 210963: 1< 0 - - > 2< 864104448 0 0 0x0 0x0 | 00000000640821b9 0000000006873caa > 3< 0 0 > 4< 0 0 8 484536320 - | - - + - > 5< - - + - | 0 - | 36 - -> 00000080:00200000:3.0:1787827173.616369:0:27439:0:(rw26.c:933:ll_write_end()) end page@00000000a7cde2eb 00000080:00200000:3.0:1787827173.616373:0:27439:0:(rw26.c:933:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1787827173.616378:0:27439:0:(vvp_io.c:1214:vvp_io_write_commit()) commit async pages: 1, from 0, to 1024 prio 0 00000008:00010000:3.0:1787827173.616392:0:27439:0:(osc_cache.c:565:osc_extent_merge()) ### extent: 00000000e070090a ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.616407:0:27439:0:(osc_cache.c:497:__osc_extent_remove()) ### extent: 00000000e070090a ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.616418:0:27439:0:(osc_cache.c:582:osc_extent_merge()) ### extent: 000000001063fa34 ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.616432:0:27439:0:(osc_cache.c:846:osc_extent_find()) ### extent: 000000001063fa34 ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.616450:0:27439:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff9e35d0e392f8, count 1, dirtied 1 00000080:00200000:3.0:1787827173.616456:0:27439:0:(vvp_io.c:1237:vvp_io_write_commit()) Committed 1 pages 1024 bytes, tot: 1024 00000080:00200000:3.0:1787827173.616460:0:27439:0:(vvp_io.c:1421:vvp_io_write_start()) oosfile: write bytes 0, result: 1024 00000020:00010000:3.0:1787827173.616470:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_release():245 00000020:00010000:3.0:1787827173.616478:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1787827173.616483:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c7d81d30 (ffff9e35c614ce18/1) at cl_lock_cancel():142 00010000:00010000:3.0:1787827173.616492:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.616507:0:27439:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 21/0,0 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1787827173.616525:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_fini():72 00000020:00010000:3.0:1787827173.616531:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c7d81d30 (ffff9e35e04b4e18/1) at cl_lock_fini():72 00000080:00200000:3.0:1787827173.616538:0:27439:0:(vvp_io.c:533:vvp_io_advance()) advancing 1024 bytes 00000080:00200000:3.0:1787827173.616542:0:27439:0:(file.c:2207:ll_file_io_generic()) Range unlock [421907, 421907] 00000080:00200000:3.0:1787827173.616549:0:27439:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787827173.616557:0:27439:0:(file.c:2240:ll_file_io_generic()) oosfile: 2 io complete with rc: 0, result: 1024, restart: 0 00000080:00200000:3.0:1787827173.616564:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728132096 i_blocks 3365288 00000080:00200000:3.0:1787827173.616570:0:27439:0:(file.c:2326:ll_file_io_generic()) iot: 2, result: 1024 00000080:00200008:3.0:1787827173.616586:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728132096, count: 1024 00000080:00200000:3.0:1787827173.616594:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421907 of 1024 to 1024 bytes 00000080:00200000:3.0:1787827173.616602:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728132096, len 1024, copied 1024 00000080:00200000:3.0:1787827173.616612:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728133120 i_blocks 3365288 00000080:00200000:3.0:1787827173.616618:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.616625:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728133120, count: 1024 00000080:00200000:3.0:1787827173.616630:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421907 of 2048 to 1024 bytes 00000080:00200000:3.0:1787827173.616637:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728133120, len 1024, copied 1024 00000080:00200000:3.0:1787827173.616643:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728134144 i_blocks 3365288 00000080:00200000:3.0:1787827173.616650:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.616655:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728134144, count: 1024 00000080:00200000:3.0:1787827173.616660:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421907 of 3072 to 1024 bytes 00000080:00200000:3.0:1787827173.616666:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728134144, len 1024, copied 1024 00000080:00200000:3.0:1787827173.616672:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728135168 i_blocks 3365288 00000080:00200000:3.0:1787827173.616676:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.616683:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728135168, count: 1024 00000080:00200000:3.0:1787827173.616689:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421908 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.616693:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 0, original count 1024 00000080:00200000:3.0:1787827173.616695:0:27439:0:(file.c:2090:ll_file_io_generic()) oosfile: write ppos: 1728135168, bytes: 1024 00000080:00200000:3.0:1787827173.616697:0:27439:0:(file.c:1848:ll_io_set_mirror()) oosfile: desiginated mirror: 0 00000020:00200000:3.0:1787827173.616702:0:27439:0:(cl_io.c:218:cl_io_rw_init()) header@000000007788600a[0x110, 369461, [0x200000402:0x27:0x0] hash] 00000020:00200000:3.0:1787827173.616704:0:27439:0:(cl_io.c:218:cl_io_rw_init()) io range: 2 [1728135168, 1728136192) 0 0 00000080:00200000:3.0:1787827173.616709:0:27439:0:(vvp_io.c:1889:vvp_io_init()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1787827173.616719:0:27439:0:(file.c:2170:ll_file_io_generic()) Range lock [421908, 421908] 00020000:00200000:3.0:1787827173.616725:0:27439:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 412 chunk: [1728135168, 1728136192) 1728136192, 1024 00020000:00200000:3.0:1787827173.616730:0:27439:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1787827173.616739:0:27439:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [421908, 421908] 00000008:00010000:3.0:1787827173.616748:0:27439:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1787827173.616752:0:27439:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff9e35c7d81d30, osc lock ffff9e35c2ddcc30, flags 0x0 00010000:00010000:3.0:1787827173.616765:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 20/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.616787:0:27439:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (864108544 864112639) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 20/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.616846:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 24/0,2 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.616872:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 24/0,2 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.616891:0:27439:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 23/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.616904:0:27439:0:(vvp_io.c:1339:vvp_io_write_start()) oosfile: write [1728135168, 1728136192) append 0 append lockpos 0 00000080:00200000:3.0:1787827173.616912:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421908 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.616949:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728135168, len 1024, copied 1024 00000080:00200000:3.0:1787827173.616959:0:27439:0:(rw26.c:933:ll_write_end()) page@000000009029d81a[3 000000005ce0a915 2 1 00000000d6bbf043 0 0 421908] 00000080:00200000:3.0:1787827173.616965:0:27439:0:(rw26.c:933:ll_write_end()) vmpage @0000000076c9af72 fffffc0001001 3:0 ffff9e34c3cb31d0 421908 lru 00000080:00200000:3.0:1787827173.616973:0:27439:0:(rw26.c:933:ll_write_end()) osc-page@000000004cefa399 210964: 1< 0 - - > 2< 864108544 0 0 0x0 0x0 | 00000000640821b9 0000000006873caa > 3< 0 0 > 4< 0 0 8 484532224 - | - - + - > 5< - - + - | 0 - | 37 - -> 00000080:00200000:3.0:1787827173.616978:0:27439:0:(rw26.c:933:ll_write_end()) end page@000000009029d81a 00000080:00200000:3.0:1787827173.616981:0:27439:0:(rw26.c:933:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1787827173.616986:0:27439:0:(vvp_io.c:1214:vvp_io_write_commit()) commit async pages: 1, from 0, to 1024 prio 0 00000008:00010000:3.0:1787827173.617001:0:27439:0:(osc_cache.c:565:osc_extent_merge()) ### extent: 00000000e070090a ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.617020:0:27439:0:(osc_cache.c:497:__osc_extent_remove()) ### extent: 00000000e070090a ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.617040:0:27439:0:(osc_cache.c:582:osc_extent_merge()) ### extent: 000000001063fa34 ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.617057:0:27439:0:(osc_cache.c:846:osc_extent_find()) ### extent: 000000001063fa34 ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.617079:0:27439:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff9e35d0e392f8, count 1, dirtied 1 00000080:00200000:3.0:1787827173.617085:0:27439:0:(vvp_io.c:1237:vvp_io_write_commit()) Committed 1 pages 1024 bytes, tot: 1024 00000080:00200000:3.0:1787827173.617091:0:27439:0:(vvp_io.c:1421:vvp_io_write_start()) oosfile: write bytes 0, result: 1024 00000020:00010000:3.0:1787827173.617099:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_release():245 00000020:00010000:3.0:1787827173.617104:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1787827173.617109:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c7d81d30 (ffff9e35c614ce18/1) at cl_lock_cancel():142 00010000:00010000:3.0:1787827173.617116:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.617126:0:27439:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 21/0,0 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1787827173.617151:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_fini():72 00000020:00010000:3.0:1787827173.617156:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c7d81d30 (ffff9e35e04b4e18/1) at cl_lock_fini():72 00000080:00200000:3.0:1787827173.617165:0:27439:0:(vvp_io.c:533:vvp_io_advance()) advancing 1024 bytes 00000080:00200000:3.0:1787827173.617169:0:27439:0:(file.c:2207:ll_file_io_generic()) Range unlock [421908, 421908] 00000080:00200000:3.0:1787827173.617174:0:27439:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787827173.617180:0:27439:0:(file.c:2240:ll_file_io_generic()) oosfile: 2 io complete with rc: 0, result: 1024, restart: 0 00000080:00200000:3.0:1787827173.617189:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728136192 i_blocks 3365288 00000080:00200000:3.0:1787827173.617197:0:27439:0:(file.c:2326:ll_file_io_generic()) iot: 2, result: 1024 00000080:00200008:3.0:1787827173.617209:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728136192, count: 1024 00000080:00200000:3.0:1787827173.617215:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421908 of 1024 to 1024 bytes 00000080:00200000:3.0:1787827173.617219:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728136192, len 1024, copied 1024 00000080:00200000:3.0:1787827173.617226:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728137216 i_blocks 3365288 00000080:00200000:3.0:1787827173.617232:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.617240:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728137216, count: 1024 00000080:00200000:3.0:1787827173.617246:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421908 of 2048 to 1024 bytes 00000080:00200000:3.0:1787827173.617253:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728137216, len 1024, copied 1024 00000080:00200000:3.0:1787827173.617260:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728138240 i_blocks 3365288 00000080:00200000:3.0:1787827173.617266:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.617273:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728138240, count: 1024 00000080:00200000:3.0:1787827173.617278:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421908 of 3072 to 1024 bytes 00000080:00200000:3.0:1787827173.617283:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728138240, len 1024, copied 1024 00000080:00200000:3.0:1787827173.617291:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728139264 i_blocks 3365288 00000080:00200000:3.0:1787827173.617298:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.617305:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728139264, count: 1024 00000080:00200000:3.0:1787827173.617311:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421909 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.617317:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 0, original count 1024 00000080:00200000:3.0:1787827173.617320:0:27439:0:(file.c:2090:ll_file_io_generic()) oosfile: write ppos: 1728139264, bytes: 1024 00000080:00200000:3.0:1787827173.617323:0:27439:0:(file.c:1848:ll_io_set_mirror()) oosfile: desiginated mirror: 0 00000020:00200000:3.0:1787827173.617332:0:27439:0:(cl_io.c:218:cl_io_rw_init()) header@000000007788600a[0x110, 369462, [0x200000402:0x27:0x0] hash] 00000020:00200000:3.0:1787827173.617335:0:27439:0:(cl_io.c:218:cl_io_rw_init()) io range: 2 [1728139264, 1728140288) 0 0 00000080:00200000:3.0:1787827173.617339:0:27439:0:(vvp_io.c:1889:vvp_io_init()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1787827173.617351:0:27439:0:(file.c:2170:ll_file_io_generic()) Range lock [421909, 421909] 00020000:00200000:3.0:1787827173.617358:0:27439:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 412 chunk: [1728139264, 1728140288) 1728140288, 1024 00020000:00200000:3.0:1787827173.617365:0:27439:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1787827173.617374:0:27439:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [421909, 421909] 00000008:00010000:3.0:1787827173.617384:0:27439:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1787827173.617390:0:27439:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff9e35c7d81d30, osc lock ffff9e35c2ddcc30, flags 0x0 00010000:00010000:3.0:1787827173.617404:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 20/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.617417:0:27439:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (864112640 864116735) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 20/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.617438:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 24/0,2 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.617465:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 24/0,2 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.617479:0:27439:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 23/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.617496:0:27439:0:(vvp_io.c:1339:vvp_io_write_start()) oosfile: write [1728139264, 1728140288) append 0 append lockpos 0 00000080:00200000:3.0:1787827173.617504:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421909 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.617526:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728139264, len 1024, copied 1024 00000080:00200000:3.0:1787827173.617533:0:27439:0:(rw26.c:933:ll_write_end()) page@00000000c356550c[3 000000005ce0a915 2 1 00000000d6bbf043 0 0 421909] 00000080:00200000:3.0:1787827173.617539:0:27439:0:(rw26.c:933:ll_write_end()) vmpage @000000009143bf66 fffffc0001001 3:0 ffff9e34c3cb3658 421909 lru 00000080:00200000:3.0:1787827173.617550:0:27439:0:(rw26.c:933:ll_write_end()) osc-page@0000000096012716 210965: 1< 0 - - > 2< 864112640 0 0 0x0 0x0 | 00000000640821b9 0000000006873caa > 3< 0 0 > 4< 0 0 8 484528128 - | - - + - > 5< - - + - | 0 - | 38 - -> 00000080:00200000:3.0:1787827173.617554:0:27439:0:(rw26.c:933:ll_write_end()) end page@00000000c356550c 00000080:00200000:3.0:1787827173.617558:0:27439:0:(rw26.c:933:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1787827173.617560:0:27439:0:(vvp_io.c:1214:vvp_io_write_commit()) commit async pages: 1, from 0, to 1024 prio 0 00000008:00010000:3.0:1787827173.617571:0:27439:0:(osc_cache.c:565:osc_extent_merge()) ### extent: 00000000e070090a ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.617587:0:27439:0:(osc_cache.c:497:__osc_extent_remove()) ### extent: 00000000e070090a ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.617601:0:27439:0:(osc_cache.c:582:osc_extent_merge()) ### extent: 000000001063fa34 ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.617625:0:27439:0:(osc_cache.c:846:osc_extent_find()) ### extent: 000000001063fa34 ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.617655:0:27439:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff9e35d0e392f8, count 1, dirtied 1 00000080:00200000:3.0:1787827173.617662:0:27439:0:(vvp_io.c:1237:vvp_io_write_commit()) Committed 1 pages 1024 bytes, tot: 1024 00000080:00200000:3.0:1787827173.617666:0:27439:0:(vvp_io.c:1421:vvp_io_write_start()) oosfile: write bytes 0, result: 1024 00000020:00010000:3.0:1787827173.617675:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_release():245 00000020:00010000:3.0:1787827173.617680:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1787827173.617683:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c7d81d30 (ffff9e35c614ce18/1) at cl_lock_cancel():142 00010000:00010000:3.0:1787827173.617688:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.617706:0:27439:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 21/0,0 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1787827173.617724:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_fini():72 00000020:00010000:3.0:1787827173.617729:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c7d81d30 (ffff9e35e04b4e18/1) at cl_lock_fini():72 00000080:00200000:3.0:1787827173.617733:0:27439:0:(vvp_io.c:533:vvp_io_advance()) advancing 1024 bytes 00000080:00200000:3.0:1787827173.617737:0:27439:0:(file.c:2207:ll_file_io_generic()) Range unlock [421909, 421909] 00000080:00200000:3.0:1787827173.617740:0:27439:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787827173.617743:0:27439:0:(file.c:2240:ll_file_io_generic()) oosfile: 2 io complete with rc: 0, result: 1024, restart: 0 00000080:00200000:3.0:1787827173.617750:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728140288 i_blocks 3365288 00000080:00200000:3.0:1787827173.617754:0:27439:0:(file.c:2326:ll_file_io_generic()) iot: 2, result: 1024 00000080:00200008:3.0:1787827173.617761:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728140288, count: 1024 00000080:00200000:3.0:1787827173.617769:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421909 of 1024 to 1024 bytes 00000080:00200000:3.0:1787827173.617780:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728140288, len 1024, copied 1024 00000080:00200000:3.0:1787827173.617786:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728141312 i_blocks 3365288 00000080:00200000:3.0:1787827173.617789:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.617793:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728141312, count: 1024 00000080:00200000:3.0:1787827173.617821:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421909 of 2048 to 1024 bytes 00000080:00200000:3.0:1787827173.617825:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728141312, len 1024, copied 1024 00000080:00200000:3.0:1787827173.617831:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728142336 i_blocks 3365288 00000080:00200000:3.0:1787827173.617838:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.617846:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728142336, count: 1024 00000080:00200000:3.0:1787827173.617851:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421909 of 3072 to 1024 bytes 00000080:00200000:3.0:1787827173.617856:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728142336, len 1024, copied 1024 00000080:00200000:3.0:1787827173.617863:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728143360 i_blocks 3365288 00000080:00200000:3.0:1787827173.617868:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.617873:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728143360, count: 1024 00000080:00200000:3.0:1787827173.617879:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421910 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.617884:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 0, original count 1024 00000080:00200000:3.0:1787827173.617889:0:27439:0:(file.c:2090:ll_file_io_generic()) oosfile: write ppos: 1728143360, bytes: 1024 00000080:00200000:3.0:1787827173.617895:0:27439:0:(file.c:1848:ll_io_set_mirror()) oosfile: desiginated mirror: 0 00000020:00200000:3.0:1787827173.617903:0:27439:0:(cl_io.c:218:cl_io_rw_init()) header@000000007788600a[0x110, 369463, [0x200000402:0x27:0x0] hash] 00000020:00200000:3.0:1787827173.617906:0:27439:0:(cl_io.c:218:cl_io_rw_init()) io range: 2 [1728143360, 1728144384) 0 0 00000080:00200000:3.0:1787827173.617911:0:27439:0:(vvp_io.c:1889:vvp_io_init()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1787827173.617935:0:27439:0:(file.c:2170:ll_file_io_generic()) Range lock [421910, 421910] 00020000:00200000:3.0:1787827173.617945:0:27439:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 412 chunk: [1728143360, 1728144384) 1728144384, 1024 00020000:00200000:3.0:1787827173.617951:0:27439:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1787827173.617960:0:27439:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [421910, 421910] 00000008:00010000:3.0:1787827173.617970:0:27439:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1787827173.617974:0:27439:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff9e35c7d81d30, osc lock ffff9e35c2ddcc30, flags 0x0 00010000:00010000:3.0:1787827173.617983:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 20/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.617992:0:27439:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (864116736 864120831) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 20/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.618009:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 24/0,2 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.618034:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 24/0,2 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.618044:0:27439:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 23/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.618055:0:27439:0:(vvp_io.c:1339:vvp_io_write_start()) oosfile: write [1728143360, 1728144384) append 0 append lockpos 0 00000080:00200000:3.0:1787827173.618062:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421910 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.618079:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728143360, len 1024, copied 1024 00000080:00200000:3.0:1787827173.618085:0:27439:0:(rw26.c:933:ll_write_end()) page@00000000adca6a5a[3 000000005ce0a915 2 1 00000000d6bbf043 0 0 421910] 00000080:00200000:3.0:1787827173.618090:0:27439:0:(rw26.c:933:ll_write_end()) vmpage @000000008d14b171 fffffc0001001 3:0 ffff9e34c3cb33a0 421910 lru 00000080:00200000:3.0:1787827173.618096:0:27439:0:(rw26.c:933:ll_write_end()) osc-page@0000000002ce278c 210966: 1< 0 - - > 2< 864116736 0 0 0x0 0x0 | 00000000640821b9 0000000006873caa > 3< 0 0 > 4< 0 0 8 484524032 - | - - + - > 5< - - + - | 0 - | 39 - -> 00000080:00200000:3.0:1787827173.618100:0:27439:0:(rw26.c:933:ll_write_end()) end page@00000000adca6a5a 00000080:00200000:3.0:1787827173.618102:0:27439:0:(rw26.c:933:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1787827173.618105:0:27439:0:(vvp_io.c:1214:vvp_io_write_commit()) commit async pages: 1, from 0, to 1024 prio 0 00000008:00010000:3.0:1787827173.618115:0:27439:0:(osc_cache.c:565:osc_extent_merge()) ### extent: 00000000e070090a ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.618133:0:27439:0:(osc_cache.c:497:__osc_extent_remove()) ### extent: 00000000e070090a ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.618151:0:27439:0:(osc_cache.c:582:osc_extent_merge()) ### extent: 000000001063fa34 ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.618167:0:27439:0:(osc_cache.c:846:osc_extent_find()) ### extent: 000000001063fa34 ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.618185:0:27439:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff9e35d0e392f8, count 1, dirtied 1 00000080:00200000:3.0:1787827173.618190:0:27439:0:(vvp_io.c:1237:vvp_io_write_commit()) Committed 1 pages 1024 bytes, tot: 1024 00000080:00200000:3.0:1787827173.618193:0:27439:0:(vvp_io.c:1421:vvp_io_write_start()) oosfile: write bytes 0, result: 1024 00000020:00010000:3.0:1787827173.618201:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_release():245 00000020:00010000:3.0:1787827173.618208:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1787827173.618211:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c7d81d30 (ffff9e35c614ce18/1) at cl_lock_cancel():142 00010000:00010000:3.0:1787827173.618217:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.618236:0:27439:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 21/0,0 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1787827173.618246:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_fini():72 00000020:00010000:3.0:1787827173.618251:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c7d81d30 (ffff9e35e04b4e18/1) at cl_lock_fini():72 00000080:00200000:3.0:1787827173.618257:0:27439:0:(vvp_io.c:533:vvp_io_advance()) advancing 1024 bytes 00000080:00200000:3.0:1787827173.618261:0:27439:0:(file.c:2207:ll_file_io_generic()) Range unlock [421910, 421910] 00000080:00200000:3.0:1787827173.618265:0:27439:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787827173.618272:0:27439:0:(file.c:2240:ll_file_io_generic()) oosfile: 2 io complete with rc: 0, result: 1024, restart: 0 00000080:00200000:3.0:1787827173.618280:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728144384 i_blocks 3365288 00000080:00200000:3.0:1787827173.618285:0:27439:0:(file.c:2326:ll_file_io_generic()) iot: 2, result: 1024 00000080:00200008:3.0:1787827173.618293:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728144384, count: 1024 00000080:00200000:3.0:1787827173.618298:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421910 of 1024 to 1024 bytes 00000080:00200000:3.0:1787827173.618302:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728144384, len 1024, copied 1024 00000080:00200000:3.0:1787827173.618306:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728145408 i_blocks 3365288 00000080:00200000:3.0:1787827173.618309:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.618312:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728145408, count: 1024 00000080:00200000:3.0:1787827173.618315:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421910 of 2048 to 1024 bytes 00000080:00200000:3.0:1787827173.618318:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728145408, len 1024, copied 1024 00000080:00200000:3.0:1787827173.618320:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728146432 i_blocks 3365288 00000080:00200000:3.0:1787827173.618322:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.618325:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728146432, count: 1024 00000080:00200000:3.0:1787827173.618332:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421910 of 3072 to 1024 bytes 00000080:00200000:3.0:1787827173.618336:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728146432, len 1024, copied 1024 00000080:00200000:3.0:1787827173.618340:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728147456 i_blocks 3365288 00000080:00200000:3.0:1787827173.618346:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.618349:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728147456, count: 1024 00000080:00200000:3.0:1787827173.618354:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421911 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.618357:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 0, original count 1024 00000080:00200000:3.0:1787827173.618360:0:27439:0:(file.c:2090:ll_file_io_generic()) oosfile: write ppos: 1728147456, bytes: 1024 00000080:00200000:3.0:1787827173.618363:0:27439:0:(file.c:1848:ll_io_set_mirror()) oosfile: desiginated mirror: 0 00000020:00200000:3.0:1787827173.618369:0:27439:0:(cl_io.c:218:cl_io_rw_init()) header@000000007788600a[0x110, 369464, [0x200000402:0x27:0x0] hash] 00000020:00200000:3.0:1787827173.618372:0:27439:0:(cl_io.c:218:cl_io_rw_init()) io range: 2 [1728147456, 1728148480) 0 0 00000080:00200000:3.0:1787827173.618375:0:27439:0:(vvp_io.c:1889:vvp_io_init()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1787827173.618380:0:27439:0:(file.c:2170:ll_file_io_generic()) Range lock [421911, 421911] 00020000:00200000:3.0:1787827173.618384:0:27439:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 412 chunk: [1728147456, 1728148480) 1728148480, 1024 00020000:00200000:3.0:1787827173.618386:0:27439:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1787827173.618390:0:27439:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [421911, 421911] 00000008:00010000:3.0:1787827173.618397:0:27439:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1787827173.618400:0:27439:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff9e35c7d81d30, osc lock ffff9e35c2ddcc30, flags 0x0 00010000:00010000:3.0:1787827173.618407:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 20/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.618425:0:27439:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (864120832 864124927) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 20/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.618446:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 24/0,2 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.618469:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 24/0,2 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.618484:0:27439:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 23/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.618498:0:27439:0:(vvp_io.c:1339:vvp_io_write_start()) oosfile: write [1728147456, 1728148480) append 0 append lockpos 0 00000080:00200000:3.0:1787827173.618505:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421911 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.618527:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728147456, len 1024, copied 1024 00000080:00200000:3.0:1787827173.618539:0:27439:0:(rw26.c:933:ll_write_end()) page@0000000061b7967d[3 000000005ce0a915 2 1 00000000d6bbf043 0 0 421911] 00000080:00200000:3.0:1787827173.618544:0:27439:0:(rw26.c:933:ll_write_end()) vmpage @00000000924eb32d fffffc0001001 3:0 ffff9e34c3cb3d98 421911 lru 00000080:00200000:3.0:1787827173.618554:0:27439:0:(rw26.c:933:ll_write_end()) osc-page@00000000cec8d80d 210967: 1< 0 - - > 2< 864120832 0 0 0x0 0x0 | 00000000640821b9 0000000006873caa > 3< 0 0 > 4< 0 0 8 484519936 - | - - + - > 5< - - + - | 0 - | 40 - -> 00000080:00200000:3.0:1787827173.618559:0:27439:0:(rw26.c:933:ll_write_end()) end page@0000000061b7967d 00000080:00200000:3.0:1787827173.618563:0:27439:0:(rw26.c:933:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1787827173.618567:0:27439:0:(vvp_io.c:1214:vvp_io_write_commit()) commit async pages: 1, from 0, to 1024 prio 0 00000008:00010000:3.0:1787827173.618579:0:27439:0:(osc_cache.c:565:osc_extent_merge()) ### extent: 00000000e070090a ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.618595:0:27439:0:(osc_cache.c:497:__osc_extent_remove()) ### extent: 00000000e070090a ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.618609:0:27439:0:(osc_cache.c:582:osc_extent_merge()) ### extent: 000000001063fa34 ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.618625:0:27439:0:(osc_cache.c:846:osc_extent_find()) ### extent: 000000001063fa34 ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.618645:0:27439:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff9e35d0e392f8, count 1, dirtied 1 00000080:00200000:3.0:1787827173.618651:0:27439:0:(vvp_io.c:1237:vvp_io_write_commit()) Committed 1 pages 1024 bytes, tot: 1024 00000080:00200000:3.0:1787827173.618656:0:27439:0:(vvp_io.c:1421:vvp_io_write_start()) oosfile: write bytes 0, result: 1024 00000020:00010000:3.0:1787827173.618666:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_release():245 00000020:00010000:3.0:1787827173.618673:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1787827173.618677:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c7d81d30 (ffff9e35c614ce18/1) at cl_lock_cancel():142 00010000:00010000:3.0:1787827173.618685:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.618707:0:27439:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 21/0,0 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1787827173.618728:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_fini():72 00000020:00010000:3.0:1787827173.618732:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c7d81d30 (ffff9e35e04b4e18/1) at cl_lock_fini():72 00000080:00200000:3.0:1787827173.618740:0:27439:0:(vvp_io.c:533:vvp_io_advance()) advancing 1024 bytes 00000080:00200000:3.0:1787827173.618747:0:27439:0:(file.c:2207:ll_file_io_generic()) Range unlock [421911, 421911] 00000080:00200000:3.0:1787827173.618753:0:27439:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787827173.618758:0:27439:0:(file.c:2240:ll_file_io_generic()) oosfile: 2 io complete with rc: 0, result: 1024, restart: 0 00000080:00200000:3.0:1787827173.618766:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728148480 i_blocks 3365288 00000080:00200000:3.0:1787827173.618771:0:27439:0:(file.c:2326:ll_file_io_generic()) iot: 2, result: 1024 00000080:00200008:3.0:1787827173.618780:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728148480, count: 1024 00000080:00200000:3.0:1787827173.618788:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421911 of 1024 to 1024 bytes 00000080:00200000:3.0:1787827173.618794:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728148480, len 1024, copied 1024 00000080:00200000:3.0:1787827173.618824:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728149504 i_blocks 3365288 00000080:00200000:3.0:1787827173.618827:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.618833:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728149504, count: 1024 00000080:00200000:3.0:1787827173.618837:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421911 of 2048 to 1024 bytes 00000080:00200000:3.0:1787827173.618841:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728149504, len 1024, copied 1024 00000080:00200000:3.0:1787827173.618847:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728150528 i_blocks 3365288 00000080:00200000:3.0:1787827173.618852:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.618857:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728150528, count: 1024 00000080:00200000:3.0:1787827173.618866:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421911 of 3072 to 1024 bytes 00000080:00200000:3.0:1787827173.618870:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728150528, len 1024, copied 1024 00000080:00200000:3.0:1787827173.618876:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728151552 i_blocks 3365288 00000080:00200000:3.0:1787827173.618883:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.618889:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728151552, count: 1024 00000080:00200000:3.0:1787827173.618898:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421912 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.618903:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 0, original count 1024 00000080:00200000:3.0:1787827173.618908:0:27439:0:(file.c:2090:ll_file_io_generic()) oosfile: write ppos: 1728151552, bytes: 1024 00000080:00200000:3.0:1787827173.618912:0:27439:0:(file.c:1848:ll_io_set_mirror()) oosfile: desiginated mirror: 0 00000020:00200000:3.0:1787827173.618931:0:27439:0:(cl_io.c:218:cl_io_rw_init()) header@000000007788600a[0x110, 369465, [0x200000402:0x27:0x0] hash] 00000020:00200000:3.0:1787827173.618933:0:27439:0:(cl_io.c:218:cl_io_rw_init()) io range: 2 [1728151552, 1728152576) 0 0 00000080:00200000:3.0:1787827173.618935:0:27439:0:(vvp_io.c:1889:vvp_io_init()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1787827173.618942:0:27439:0:(file.c:2170:ll_file_io_generic()) Range lock [421912, 421912] 00020000:00200000:3.0:1787827173.618945:0:27439:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 412 chunk: [1728151552, 1728152576) 1728152576, 1024 00020000:00200000:3.0:1787827173.618948:0:27439:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1787827173.618955:0:27439:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [421912, 421912] 00000008:00010000:3.0:1787827173.618961:0:27439:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1787827173.618964:0:27439:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff9e35c7d81d30, osc lock ffff9e35c2ddcc30, flags 0x0 00010000:00010000:3.0:1787827173.618974:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 20/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.618984:0:27439:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (864124928 864129023) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 20/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.619003:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 24/0,2 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.619031:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 24/0,2 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.619041:0:27439:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 23/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.619047:0:27439:0:(vvp_io.c:1339:vvp_io_write_start()) oosfile: write [1728151552, 1728152576) append 0 append lockpos 0 00000080:00200000:3.0:1787827173.619051:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421912 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.619069:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728151552, len 1024, copied 1024 00000080:00200000:3.0:1787827173.619078:0:27439:0:(rw26.c:933:ll_write_end()) page@000000007ac95a39[3 000000005ce0a915 2 1 00000000d6bbf043 0 0 421912] 00000080:00200000:3.0:1787827173.619083:0:27439:0:(rw26.c:933:ll_write_end()) vmpage @00000000c54136c3 fffffc0001001 3:0 ffff9e34c3cb3000 421912 lru 00000080:00200000:3.0:1787827173.619090:0:27439:0:(rw26.c:933:ll_write_end()) osc-page@00000000678bc860 210968: 1< 0 - - > 2< 864124928 0 0 0x0 0x0 | 00000000640821b9 0000000006873caa > 3< 0 0 > 4< 0 0 8 484515840 - | - - + - > 5< - - + - | 0 - | 41 - -> 00000080:00200000:3.0:1787827173.619092:0:27439:0:(rw26.c:933:ll_write_end()) end page@000000007ac95a39 00000080:00200000:3.0:1787827173.619095:0:27439:0:(rw26.c:933:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1787827173.619097:0:27439:0:(vvp_io.c:1214:vvp_io_write_commit()) commit async pages: 1, from 0, to 1024 prio 0 00000008:00010000:3.0:1787827173.619107:0:27439:0:(osc_cache.c:565:osc_extent_merge()) ### extent: 00000000e070090a ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.619115:0:27439:0:(osc_cache.c:497:__osc_extent_remove()) ### extent: 00000000e070090a ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.619125:0:27439:0:(osc_cache.c:582:osc_extent_merge()) ### extent: 000000001063fa34 ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.619135:0:27439:0:(osc_cache.c:846:osc_extent_find()) ### extent: 000000001063fa34 ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.619157:0:27439:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff9e35d0e392f8, count 1, dirtied 1 00000080:00200000:3.0:1787827173.619164:0:27439:0:(vvp_io.c:1237:vvp_io_write_commit()) Committed 1 pages 1024 bytes, tot: 1024 00000080:00200000:3.0:1787827173.619168:0:27439:0:(vvp_io.c:1421:vvp_io_write_start()) oosfile: write bytes 0, result: 1024 00000020:00010000:3.0:1787827173.619178:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_release():245 00000020:00010000:3.0:1787827173.619183:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1787827173.619190:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c7d81d30 (ffff9e35c614ce18/1) at cl_lock_cancel():142 00010000:00010000:3.0:1787827173.619200:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.619213:0:27439:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 21/0,0 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1787827173.619228:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_fini():72 00000020:00010000:3.0:1787827173.619231:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c7d81d30 (ffff9e35e04b4e18/1) at cl_lock_fini():72 00000080:00200000:3.0:1787827173.619239:0:27439:0:(vvp_io.c:533:vvp_io_advance()) advancing 1024 bytes 00000080:00200000:3.0:1787827173.619242:0:27439:0:(file.c:2207:ll_file_io_generic()) Range unlock [421912, 421912] 00000080:00200000:3.0:1787827173.619247:0:27439:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787827173.619250:0:27439:0:(file.c:2240:ll_file_io_generic()) oosfile: 2 io complete with rc: 0, result: 1024, restart: 0 00000080:00200000:3.0:1787827173.619258:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728152576 i_blocks 3365288 00000080:00200000:3.0:1787827173.619262:0:27439:0:(file.c:2326:ll_file_io_generic()) iot: 2, result: 1024 00000080:00200008:3.0:1787827173.619270:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728152576, count: 1024 00000080:00200000:3.0:1787827173.619276:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421912 of 1024 to 1024 bytes 00000080:00200000:3.0:1787827173.619281:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728152576, len 1024, copied 1024 00000080:00200000:3.0:1787827173.619288:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728153600 i_blocks 3365288 00000080:00200000:3.0:1787827173.619292:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.619296:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728153600, count: 1024 00000080:00200000:3.0:1787827173.619302:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421912 of 2048 to 1024 bytes 00000080:00200000:3.0:1787827173.619307:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728153600, len 1024, copied 1024 00000080:00200000:3.0:1787827173.619321:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728154624 i_blocks 3365288 00000080:00200000:3.0:1787827173.619325:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.619329:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728154624, count: 1024 00000080:00200000:3.0:1787827173.619334:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421912 of 3072 to 1024 bytes 00000080:00200000:3.0:1787827173.619336:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728154624, len 1024, copied 1024 00000080:00200000:3.0:1787827173.619339:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728155648 i_blocks 3365288 00000080:00200000:3.0:1787827173.619345:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.619350:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728155648, count: 1024 00000080:00200000:3.0:1787827173.619355:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421913 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.619361:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 0, original count 1024 00000080:00200000:3.0:1787827173.619362:0:27439:0:(file.c:2090:ll_file_io_generic()) oosfile: write ppos: 1728155648, bytes: 1024 00000080:00200000:3.0:1787827173.619364:0:27439:0:(file.c:1848:ll_io_set_mirror()) oosfile: desiginated mirror: 0 00000020:00200000:3.0:1787827173.619367:0:27439:0:(cl_io.c:218:cl_io_rw_init()) header@000000007788600a[0x110, 369466, [0x200000402:0x27:0x0] hash] 00000020:00200000:3.0:1787827173.619369:0:27439:0:(cl_io.c:218:cl_io_rw_init()) io range: 2 [1728155648, 1728156672) 0 0 00000080:00200000:3.0:1787827173.619373:0:27439:0:(vvp_io.c:1889:vvp_io_init()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1787827173.619378:0:27439:0:(file.c:2170:ll_file_io_generic()) Range lock [421913, 421913] 00020000:00200000:3.0:1787827173.619381:0:27439:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 412 chunk: [1728155648, 1728156672) 1728156672, 1024 00020000:00200000:3.0:1787827173.619384:0:27439:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1787827173.619391:0:27439:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [421913, 421913] 00000008:00010000:3.0:1787827173.619397:0:27439:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1787827173.619400:0:27439:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff9e35c7d81d30, osc lock ffff9e35c2ddcc30, flags 0x0 00010000:00010000:3.0:1787827173.619408:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 20/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.619422:0:27439:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (864129024 864133119) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 20/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.619435:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 24/0,2 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.619447:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 24/0,2 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.619459:0:27439:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 23/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.619471:0:27439:0:(vvp_io.c:1339:vvp_io_write_start()) oosfile: write [1728155648, 1728156672) append 0 append lockpos 0 00000080:00200000:3.0:1787827173.619479:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421913 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.619509:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728155648, len 1024, copied 1024 00000080:00200000:3.0:1787827173.619520:0:27439:0:(rw26.c:933:ll_write_end()) page@00000000ebc4abbe[3 000000005ce0a915 2 1 00000000d6bbf043 0 0 421913] 00000080:00200000:3.0:1787827173.619528:0:27439:0:(rw26.c:933:ll_write_end()) vmpage @0000000027058a3c fffffc0001001 3:0 ffff9e34c3cb4000 421913 lru 00000080:00200000:3.0:1787827173.619544:0:27439:0:(rw26.c:933:ll_write_end()) osc-page@0000000013c48fc7 210969: 1< 0 - - > 2< 864129024 0 0 0x0 0x0 | 00000000640821b9 0000000006873caa > 3< 0 0 > 4< 0 0 8 484511744 - | - - + - > 5< - - + - | 0 - | 42 - -> 00000080:00200000:3.0:1787827173.619549:0:27439:0:(rw26.c:933:ll_write_end()) end page@00000000ebc4abbe 00000080:00200000:3.0:1787827173.619550:0:27439:0:(rw26.c:933:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1787827173.619552:0:27439:0:(vvp_io.c:1214:vvp_io_write_commit()) commit async pages: 1, from 0, to 1024 prio 0 00000008:00010000:3.0:1787827173.619562:0:27439:0:(osc_cache.c:565:osc_extent_merge()) ### extent: 00000000e070090a ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.619572:0:27439:0:(osc_cache.c:497:__osc_extent_remove()) ### extent: 00000000e070090a ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.619580:0:27439:0:(osc_cache.c:582:osc_extent_merge()) ### extent: 000000001063fa34 ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.619585:0:27439:0:(osc_cache.c:846:osc_extent_find()) ### extent: 000000001063fa34 ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.619594:0:27439:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff9e35d0e392f8, count 1, dirtied 1 00000080:00200000:3.0:1787827173.619596:0:27439:0:(vvp_io.c:1237:vvp_io_write_commit()) Committed 1 pages 1024 bytes, tot: 1024 00000080:00200000:3.0:1787827173.619598:0:27439:0:(vvp_io.c:1421:vvp_io_write_start()) oosfile: write bytes 0, result: 1024 00000020:00010000:3.0:1787827173.619603:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_release():245 00000020:00010000:3.0:1787827173.619608:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1787827173.619612:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c7d81d30 (ffff9e35c614ce18/1) at cl_lock_cancel():142 00010000:00010000:3.0:1787827173.619619:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.619639:0:27439:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 21/0,0 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1787827173.619653:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_fini():72 00000020:00010000:3.0:1787827173.619655:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c7d81d30 (ffff9e35e04b4e18/1) at cl_lock_fini():72 00000080:00200000:3.0:1787827173.619658:0:27439:0:(vvp_io.c:533:vvp_io_advance()) advancing 1024 bytes 00000080:00200000:3.0:1787827173.619663:0:27439:0:(file.c:2207:ll_file_io_generic()) Range unlock [421913, 421913] 00000080:00200000:3.0:1787827173.619669:0:27439:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787827173.619679:0:27439:0:(file.c:2240:ll_file_io_generic()) oosfile: 2 io complete with rc: 0, result: 1024, restart: 0 00000080:00200000:3.0:1787827173.619687:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728156672 i_blocks 3365288 00000080:00200000:3.0:1787827173.619693:0:27439:0:(file.c:2326:ll_file_io_generic()) iot: 2, result: 1024 00000080:00200008:3.0:1787827173.619702:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728156672, count: 1024 00000080:00200000:3.0:1787827173.619709:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421913 of 1024 to 1024 bytes 00000080:00200000:3.0:1787827173.619715:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728156672, len 1024, copied 1024 00000080:00200000:3.0:1787827173.619724:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728157696 i_blocks 3365288 00000080:00200000:3.0:1787827173.619727:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.619731:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728157696, count: 1024 00000080:00200000:3.0:1787827173.619735:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421913 of 2048 to 1024 bytes 00000080:00200000:3.0:1787827173.619739:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728157696, len 1024, copied 1024 00000080:00200000:3.0:1787827173.619745:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728158720 i_blocks 3365288 00000080:00200000:3.0:1787827173.619749:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.619756:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728158720, count: 1024 00000080:00200000:3.0:1787827173.619763:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421913 of 3072 to 1024 bytes 00000080:00200000:3.0:1787827173.619769:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728158720, len 1024, copied 1024 00000080:00200000:3.0:1787827173.619776:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728159744 i_blocks 3365288 00000080:00200000:3.0:1787827173.619783:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.619790:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728159744, count: 1024 00000080:00200000:3.0:1787827173.619819:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421914 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.619823:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 0, original count 1024 00000080:00200000:3.0:1787827173.619825:0:27439:0:(file.c:2090:ll_file_io_generic()) oosfile: write ppos: 1728159744, bytes: 1024 00000080:00200000:3.0:1787827173.619829:0:27439:0:(file.c:1848:ll_io_set_mirror()) oosfile: desiginated mirror: 0 00000020:00200000:3.0:1787827173.619834:0:27439:0:(cl_io.c:218:cl_io_rw_init()) header@000000007788600a[0x110, 369467, [0x200000402:0x27:0x0] hash] 00000020:00200000:3.0:1787827173.619837:0:27439:0:(cl_io.c:218:cl_io_rw_init()) io range: 2 [1728159744, 1728160768) 0 0 00000080:00200000:3.0:1787827173.619839:0:27439:0:(vvp_io.c:1889:vvp_io_init()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1787827173.619844:0:27439:0:(file.c:2170:ll_file_io_generic()) Range lock [421914, 421914] 00020000:00200000:3.0:1787827173.619849:0:27439:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 412 chunk: [1728159744, 1728160768) 1728160768, 1024 00020000:00200000:3.0:1787827173.619851:0:27439:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1787827173.619857:0:27439:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [421914, 421914] 00000008:00010000:3.0:1787827173.619863:0:27439:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1787827173.619866:0:27439:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff9e35c7d81d30, osc lock ffff9e35c2ddcc30, flags 0x0 00010000:00010000:3.0:1787827173.619874:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 20/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.619891:0:27439:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (864133120 864137215) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 20/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.619910:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 24/0,2 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.619946:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 24/0,2 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.619956:0:27439:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 23/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.619964:0:27439:0:(vvp_io.c:1339:vvp_io_write_start()) oosfile: write [1728159744, 1728160768) append 0 append lockpos 0 00000080:00200000:3.0:1787827173.619970:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421914 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.619987:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728159744, len 1024, copied 1024 00000080:00200000:3.0:1787827173.619993:0:27439:0:(rw26.c:933:ll_write_end()) page@000000008c4ef2aa[3 000000005ce0a915 2 1 00000000d6bbf043 0 0 421914] 00000080:00200000:3.0:1787827173.619996:0:27439:0:(rw26.c:933:ll_write_end()) vmpage @00000000f8176aeb fffffc0001001 3:0 ffff9e34c3cb42b8 421914 lru 00000080:00200000:3.0:1787827173.620004:0:27439:0:(rw26.c:933:ll_write_end()) osc-page@000000006110f45f 210970: 1< 0 - - > 2< 864133120 0 0 0x0 0x0 | 00000000640821b9 0000000006873caa > 3< 0 0 > 4< 0 0 8 484507648 - | - - + - > 5< - - + - | 0 - | 43 - -> 00000080:00200000:3.0:1787827173.620008:0:27439:0:(rw26.c:933:ll_write_end()) end page@000000008c4ef2aa 00000080:00200000:3.0:1787827173.620011:0:27439:0:(rw26.c:933:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1787827173.620013:0:27439:0:(vvp_io.c:1214:vvp_io_write_commit()) commit async pages: 1, from 0, to 1024 prio 0 00000008:00010000:3.0:1787827173.620020:0:27439:0:(osc_cache.c:565:osc_extent_merge()) ### extent: 00000000e070090a ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.620027:0:27439:0:(osc_cache.c:497:__osc_extent_remove()) ### extent: 00000000e070090a ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.620039:0:27439:0:(osc_cache.c:582:osc_extent_merge()) ### extent: 000000001063fa34 ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.620056:0:27439:0:(osc_cache.c:846:osc_extent_find()) ### extent: 000000001063fa34 ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.620078:0:27439:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff9e35d0e392f8, count 1, dirtied 1 00000080:00200000:3.0:1787827173.620083:0:27439:0:(vvp_io.c:1237:vvp_io_write_commit()) Committed 1 pages 1024 bytes, tot: 1024 00000080:00200000:3.0:1787827173.620088:0:27439:0:(vvp_io.c:1421:vvp_io_write_start()) oosfile: write bytes 0, result: 1024 00000020:00010000:3.0:1787827173.620098:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_release():245 00000020:00010000:3.0:1787827173.620106:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1787827173.620113:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c7d81d30 (ffff9e35c614ce18/1) at cl_lock_cancel():142 00010000:00010000:3.0:1787827173.620121:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.620131:0:27439:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 21/0,0 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1787827173.620142:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_fini():72 00000020:00010000:3.0:1787827173.620145:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c7d81d30 (ffff9e35e04b4e18/1) at cl_lock_fini():72 00000080:00200000:3.0:1787827173.620148:0:27439:0:(vvp_io.c:533:vvp_io_advance()) advancing 1024 bytes 00000080:00200000:3.0:1787827173.620151:0:27439:0:(file.c:2207:ll_file_io_generic()) Range unlock [421914, 421914] 00000080:00200000:3.0:1787827173.620154:0:27439:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787827173.620156:0:27439:0:(file.c:2240:ll_file_io_generic()) oosfile: 2 io complete with rc: 0, result: 1024, restart: 0 00000080:00200000:3.0:1787827173.620161:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728160768 i_blocks 3365288 00000080:00200000:3.0:1787827173.620163:0:27439:0:(file.c:2326:ll_file_io_generic()) iot: 2, result: 1024 00000080:00200008:3.0:1787827173.620172:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728160768, count: 1024 00000080:00200000:3.0:1787827173.620179:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421914 of 1024 to 1024 bytes 00000080:00200000:3.0:1787827173.620182:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728160768, len 1024, copied 1024 00000080:00200000:3.0:1787827173.620188:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728161792 i_blocks 3365288 00000080:00200000:3.0:1787827173.620192:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.620197:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728161792, count: 1024 00000080:00200000:3.0:1787827173.620202:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421914 of 2048 to 1024 bytes 00000080:00200000:3.0:1787827173.620206:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728161792, len 1024, copied 1024 00000080:00200000:3.0:1787827173.620211:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728162816 i_blocks 3365288 00000080:00200000:3.0:1787827173.620212:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.620215:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728162816, count: 1024 00000080:00200000:3.0:1787827173.620220:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421914 of 3072 to 1024 bytes 00000080:00200000:3.0:1787827173.620225:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728162816, len 1024, copied 1024 00000080:00200000:3.0:1787827173.620233:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728163840 i_blocks 3365288 00000080:00200000:3.0:1787827173.620239:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.620246:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728163840, count: 1024 00000080:00200000:3.0:1787827173.620255:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421915 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.620260:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 0, original count 1024 00000080:00200000:3.0:1787827173.620264:0:27439:0:(file.c:2090:ll_file_io_generic()) oosfile: write ppos: 1728163840, bytes: 1024 00000080:00200000:3.0:1787827173.620269:0:27439:0:(file.c:1848:ll_io_set_mirror()) oosfile: desiginated mirror: 0 00000020:00200000:3.0:1787827173.620276:0:27439:0:(cl_io.c:218:cl_io_rw_init()) header@000000007788600a[0x110, 369468, [0x200000402:0x27:0x0] hash] 00000020:00200000:3.0:1787827173.620279:0:27439:0:(cl_io.c:218:cl_io_rw_init()) io range: 2 [1728163840, 1728164864) 0 0 00000080:00200000:3.0:1787827173.620284:0:27439:0:(vvp_io.c:1889:vvp_io_init()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1787827173.620290:0:27439:0:(file.c:2170:ll_file_io_generic()) Range lock [421915, 421915] 00020000:00200000:3.0:1787827173.620296:0:27439:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 412 chunk: [1728163840, 1728164864) 1728164864, 1024 00020000:00200000:3.0:1787827173.620300:0:27439:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1787827173.620309:0:27439:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [421915, 421915] 00000008:00010000:3.0:1787827173.620318:0:27439:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1787827173.620322:0:27439:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff9e35c7d81d30, osc lock ffff9e35c2ddcc30, flags 0x0 00010000:00010000:3.0:1787827173.620334:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 20/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.620351:0:27439:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (864137216 864141311) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 20/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.620365:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 24/0,2 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.620389:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 24/0,2 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.620398:0:27439:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 23/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.620412:0:27439:0:(vvp_io.c:1339:vvp_io_write_start()) oosfile: write [1728163840, 1728164864) append 0 append lockpos 0 00000080:00200000:3.0:1787827173.620420:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421915 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.620441:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728163840, len 1024, copied 1024 00000080:00200000:3.0:1787827173.620450:0:27439:0:(rw26.c:933:ll_write_end()) page@00000000ef356eb3[3 000000005ce0a915 2 1 00000000d6bbf043 0 0 421915] 00000080:00200000:3.0:1787827173.620457:0:27439:0:(rw26.c:933:ll_write_end()) vmpage @00000000b05752ef fffffc0001001 3:0 ffff9e34c3cb4488 421915 lru 00000080:00200000:3.0:1787827173.620467:0:27439:0:(rw26.c:933:ll_write_end()) osc-page@000000009584e3cc 210971: 1< 0 - - > 2< 864137216 0 0 0x0 0x0 | 00000000640821b9 0000000006873caa > 3< 0 0 > 4< 0 0 8 484503552 - | - - + - > 5< - - + - | 0 - | 44 - -> 00000080:00200000:3.0:1787827173.620471:0:27439:0:(rw26.c:933:ll_write_end()) end page@00000000ef356eb3 00000080:00200000:3.0:1787827173.620474:0:27439:0:(rw26.c:933:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1787827173.620477:0:27439:0:(vvp_io.c:1214:vvp_io_write_commit()) commit async pages: 1, from 0, to 1024 prio 0 00000008:00010000:3.0:1787827173.620491:0:27439:0:(osc_cache.c:565:osc_extent_merge()) ### extent: 00000000e070090a ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.620511:0:27439:0:(osc_cache.c:497:__osc_extent_remove()) ### extent: 00000000e070090a ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.620528:0:27439:0:(osc_cache.c:582:osc_extent_merge()) ### extent: 000000001063fa34 ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.620541:0:27439:0:(osc_cache.c:846:osc_extent_find()) ### extent: 000000001063fa34 ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.620558:0:27439:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff9e35d0e392f8, count 1, dirtied 1 00000080:00200000:3.0:1787827173.620563:0:27439:0:(vvp_io.c:1237:vvp_io_write_commit()) Committed 1 pages 1024 bytes, tot: 1024 00000080:00200000:3.0:1787827173.620567:0:27439:0:(vvp_io.c:1421:vvp_io_write_start()) oosfile: write bytes 0, result: 1024 00000020:00010000:3.0:1787827173.620574:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_release():245 00000020:00010000:3.0:1787827173.620577:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1787827173.620583:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c7d81d30 (ffff9e35c614ce18/1) at cl_lock_cancel():142 00010000:00010000:3.0:1787827173.620590:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.620609:0:27439:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 21/0,0 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1787827173.620616:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_fini():72 00000020:00010000:3.0:1787827173.620621:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c7d81d30 (ffff9e35e04b4e18/1) at cl_lock_fini():72 00000080:00200000:3.0:1787827173.620625:0:27439:0:(vvp_io.c:533:vvp_io_advance()) advancing 1024 bytes 00000080:00200000:3.0:1787827173.620629:0:27439:0:(file.c:2207:ll_file_io_generic()) Range unlock [421915, 421915] 00000080:00200000:3.0:1787827173.620634:0:27439:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787827173.620642:0:27439:0:(file.c:2240:ll_file_io_generic()) oosfile: 2 io complete with rc: 0, result: 1024, restart: 0 00000080:00200000:3.0:1787827173.620650:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728164864 i_blocks 3365288 00000080:00200000:3.0:1787827173.620656:0:27439:0:(file.c:2326:ll_file_io_generic()) iot: 2, result: 1024 00000080:00200008:3.0:1787827173.620664:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728164864, count: 1024 00000080:00200000:3.0:1787827173.620671:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421915 of 1024 to 1024 bytes 00000080:00200000:3.0:1787827173.620675:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728164864, len 1024, copied 1024 00000080:00200000:3.0:1787827173.620681:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728165888 i_blocks 3365288 00000080:00200000:3.0:1787827173.620688:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.620696:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728165888, count: 1024 00000080:00200000:3.0:1787827173.620705:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421915 of 2048 to 1024 bytes 00000080:00200000:3.0:1787827173.620711:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728165888, len 1024, copied 1024 00000080:00200000:3.0:1787827173.620719:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728166912 i_blocks 3365288 00000080:00200000:3.0:1787827173.620724:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.620729:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728166912, count: 1024 00000080:00200000:3.0:1787827173.620735:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421915 of 3072 to 1024 bytes 00000080:00200000:3.0:1787827173.620740:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728166912, len 1024, copied 1024 00000080:00200000:3.0:1787827173.620746:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728167936 i_blocks 3365288 00000080:00200000:3.0:1787827173.620754:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.620761:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728167936, count: 1024 00000080:00200000:3.0:1787827173.620768:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421916 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.620773:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 0, original count 1024 00000080:00200000:3.0:1787827173.620775:0:27439:0:(file.c:2090:ll_file_io_generic()) oosfile: write ppos: 1728167936, bytes: 1024 00000080:00200000:3.0:1787827173.620778:0:27439:0:(file.c:1848:ll_io_set_mirror()) oosfile: desiginated mirror: 0 00000020:00200000:3.0:1787827173.620787:0:27439:0:(cl_io.c:218:cl_io_rw_init()) header@000000007788600a[0x110, 369469, [0x200000402:0x27:0x0] hash] 00000020:00200000:3.0:1787827173.620792:0:27439:0:(cl_io.c:218:cl_io_rw_init()) io range: 2 [1728167936, 1728168960) 0 0 00000080:00200000:3.0:1787827173.620825:0:27439:0:(vvp_io.c:1889:vvp_io_init()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1787827173.620833:0:27439:0:(file.c:2170:ll_file_io_generic()) Range lock [421916, 421916] 00020000:00200000:3.0:1787827173.620837:0:27439:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 412 chunk: [1728167936, 1728168960) 1728168960, 1024 00020000:00200000:3.0:1787827173.620842:0:27439:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1787827173.620848:0:27439:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [421916, 421916] 00000008:00010000:3.0:1787827173.620854:0:27439:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1787827173.620858:0:27439:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff9e35c7d81d30, osc lock ffff9e35c2ddcc30, flags 0x0 00010000:00010000:3.0:1787827173.620867:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 20/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.620886:0:27439:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (864141312 864145407) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 20/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.620903:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 24/0,2 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.620938:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 24/0,2 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.620950:0:27439:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 23/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.620964:0:27439:0:(vvp_io.c:1339:vvp_io_write_start()) oosfile: write [1728167936, 1728168960) append 0 append lockpos 0 00000080:00200000:3.0:1787827173.620974:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421916 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.620996:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728167936, len 1024, copied 1024 00000080:00200000:3.0:1787827173.621005:0:27439:0:(rw26.c:933:ll_write_end()) page@0000000005b7a357[3 000000005ce0a915 2 1 00000000d6bbf043 0 0 421916] 00000080:00200000:3.0:1787827173.621011:0:27439:0:(rw26.c:933:ll_write_end()) vmpage @00000000a50657d9 fffffc0001001 3:0 ffff9e34c3cb4bc8 421916 lru 00000080:00200000:3.0:1787827173.621023:0:27439:0:(rw26.c:933:ll_write_end()) osc-page@00000000f5115cf7 210972: 1< 0 - - > 2< 864141312 0 0 0x0 0x0 | 00000000640821b9 0000000006873caa > 3< 0 0 > 4< 0 0 8 484499456 - | - - + - > 5< - - + - | 0 - | 45 - -> 00000080:00200000:3.0:1787827173.621026:0:27439:0:(rw26.c:933:ll_write_end()) end page@0000000005b7a357 00000080:00200000:3.0:1787827173.621029:0:27439:0:(rw26.c:933:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1787827173.621033:0:27439:0:(vvp_io.c:1214:vvp_io_write_commit()) commit async pages: 1, from 0, to 1024 prio 0 00000008:00010000:3.0:1787827173.621045:0:27439:0:(osc_cache.c:565:osc_extent_merge()) ### extent: 00000000e070090a ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.621063:0:27439:0:(osc_cache.c:497:__osc_extent_remove()) ### extent: 00000000e070090a ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.621075:0:27439:0:(osc_cache.c:582:osc_extent_merge()) ### extent: 000000001063fa34 ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.621090:0:27439:0:(osc_cache.c:846:osc_extent_find()) ### extent: 000000001063fa34 ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.621106:0:27439:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff9e35d0e392f8, count 1, dirtied 1 00000080:00200000:3.0:1787827173.621111:0:27439:0:(vvp_io.c:1237:vvp_io_write_commit()) Committed 1 pages 1024 bytes, tot: 1024 00000080:00200000:3.0:1787827173.621114:0:27439:0:(vvp_io.c:1421:vvp_io_write_start()) oosfile: write bytes 0, result: 1024 00000020:00010000:3.0:1787827173.621122:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_release():245 00000020:00010000:3.0:1787827173.621127:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1787827173.621134:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c7d81d30 (ffff9e35c614ce18/1) at cl_lock_cancel():142 00010000:00010000:3.0:1787827173.621142:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.621163:0:27439:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 21/0,0 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1787827173.621187:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_fini():72 00000020:00010000:3.0:1787827173.621193:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c7d81d30 (ffff9e35e04b4e18/1) at cl_lock_fini():72 00000080:00200000:3.0:1787827173.621200:0:27439:0:(vvp_io.c:533:vvp_io_advance()) advancing 1024 bytes 00000080:00200000:3.0:1787827173.621204:0:27439:0:(file.c:2207:ll_file_io_generic()) Range unlock [421916, 421916] 00000080:00200000:3.0:1787827173.621208:0:27439:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787827173.621211:0:27439:0:(file.c:2240:ll_file_io_generic()) oosfile: 2 io complete with rc: 0, result: 1024, restart: 0 00000080:00200000:3.0:1787827173.621216:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728168960 i_blocks 3365288 00000080:00200000:3.0:1787827173.621220:0:27439:0:(file.c:2326:ll_file_io_generic()) iot: 2, result: 1024 00000080:00200008:3.0:1787827173.621228:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728168960, count: 1024 00000080:00200000:3.0:1787827173.621235:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421916 of 1024 to 1024 bytes 00000080:00200000:3.0:1787827173.621239:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728168960, len 1024, copied 1024 00000080:00200000:3.0:1787827173.621246:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728169984 i_blocks 3365288 00000080:00200000:3.0:1787827173.621252:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.621260:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728169984, count: 1024 00000080:00200000:3.0:1787827173.621268:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421916 of 2048 to 1024 bytes 00000080:00200000:3.0:1787827173.621274:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728169984, len 1024, copied 1024 00000080:00200000:3.0:1787827173.621281:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728171008 i_blocks 3365288 00000080:00200000:3.0:1787827173.621288:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.621294:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728171008, count: 1024 00000080:00200000:3.0:1787827173.621299:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421916 of 3072 to 1024 bytes 00000080:00200000:3.0:1787827173.621304:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728171008, len 1024, copied 1024 00000080:00200000:3.0:1787827173.621311:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728172032 i_blocks 3365288 00000080:00200000:3.0:1787827173.621317:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.621324:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728172032, count: 1024 00000080:00200000:3.0:1787827173.621331:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421917 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.621336:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 0, original count 1024 00000080:00200000:3.0:1787827173.621341:0:27439:0:(file.c:2090:ll_file_io_generic()) oosfile: write ppos: 1728172032, bytes: 1024 00000080:00200000:3.0:1787827173.621346:0:27439:0:(file.c:1848:ll_io_set_mirror()) oosfile: desiginated mirror: 0 00000020:00200000:3.0:1787827173.621355:0:27439:0:(cl_io.c:218:cl_io_rw_init()) header@000000007788600a[0x110, 369470, [0x200000402:0x27:0x0] hash] 00000020:00200000:3.0:1787827173.621359:0:27439:0:(cl_io.c:218:cl_io_rw_init()) io range: 2 [1728172032, 1728173056) 0 0 00000080:00200000:3.0:1787827173.621362:0:27439:0:(vvp_io.c:1889:vvp_io_init()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1787827173.621370:0:27439:0:(file.c:2170:ll_file_io_generic()) Range lock [421917, 421917] 00020000:00200000:3.0:1787827173.621376:0:27439:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 412 chunk: [1728172032, 1728173056) 1728173056, 1024 00020000:00200000:3.0:1787827173.621381:0:27439:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1787827173.621391:0:27439:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [421917, 421917] 00000008:00010000:3.0:1787827173.621400:0:27439:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1787827173.621404:0:27439:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff9e35c7d81d30, osc lock ffff9e35c2ddcc30, flags 0x0 00010000:00010000:3.0:1787827173.621415:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 20/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.621434:0:27439:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (864145408 864149503) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 20/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.621449:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 24/0,2 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.621472:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 24/0,2 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.621483:0:27439:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 23/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.621497:0:27439:0:(vvp_io.c:1339:vvp_io_write_start()) oosfile: write [1728172032, 1728173056) append 0 append lockpos 0 00000080:00200000:3.0:1787827173.621505:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421917 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.621533:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728172032, len 1024, copied 1024 00000080:00200000:3.0:1787827173.621543:0:27439:0:(rw26.c:933:ll_write_end()) page@0000000090eacab1[3 000000005ce0a915 2 1 00000000d6bbf043 0 0 421917] 00000080:00200000:3.0:1787827173.621551:0:27439:0:(rw26.c:933:ll_write_end()) vmpage @00000000bb8c5bf7 fffffc0001021 2:0 ffff9e34c3cb49f8 421917 lru 00000080:00200000:3.0:1787827173.621563:0:27439:0:(rw26.c:933:ll_write_end()) osc-page@00000000390a2e3e 210973: 1< 0 - - > 2< 864145408 0 0 0x0 0x0 | 00000000640821b9 0000000006873caa > 3< 0 0 > 4< 0 0 8 484495360 - | - - + - > 5< - - + - | 0 - | 46 - -> 00000080:00200000:3.0:1787827173.621569:0:27439:0:(rw26.c:933:ll_write_end()) end page@0000000090eacab1 00000080:00200000:3.0:1787827173.621572:0:27439:0:(rw26.c:933:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1787827173.621577:0:27439:0:(vvp_io.c:1214:vvp_io_write_commit()) commit async pages: 1, from 0, to 1024 prio 0 00000008:00010000:3.0:1787827173.621591:0:27439:0:(osc_cache.c:565:osc_extent_merge()) ### extent: 00000000e070090a ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.621606:0:27439:0:(osc_cache.c:497:__osc_extent_remove()) ### extent: 00000000e070090a ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.621619:0:27439:0:(osc_cache.c:582:osc_extent_merge()) ### extent: 000000001063fa34 ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.621629:0:27439:0:(osc_cache.c:846:osc_extent_find()) ### extent: 000000001063fa34 ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.621647:0:27439:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff9e35d0e392f8, count 1, dirtied 1 00000080:00200000:3.0:1787827173.621653:0:27439:0:(vvp_io.c:1237:vvp_io_write_commit()) Committed 1 pages 1024 bytes, tot: 1024 00000080:00200000:3.0:1787827173.621657:0:27439:0:(vvp_io.c:1421:vvp_io_write_start()) oosfile: write bytes 0, result: 1024 00000020:00010000:3.0:1787827173.621667:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_release():245 00000020:00010000:3.0:1787827173.621674:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1787827173.621681:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c7d81d30 (ffff9e35c614ce18/1) at cl_lock_cancel():142 00010000:00010000:3.0:1787827173.621687:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.621705:0:27439:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 21/0,0 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1787827173.621730:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_fini():72 00000020:00010000:3.0:1787827173.621735:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c7d81d30 (ffff9e35e04b4e18/1) at cl_lock_fini():72 00000080:00200000:3.0:1787827173.621743:0:27439:0:(vvp_io.c:533:vvp_io_advance()) advancing 1024 bytes 00000080:00200000:3.0:1787827173.621748:0:27439:0:(file.c:2207:ll_file_io_generic()) Range unlock [421917, 421917] 00000080:00200000:3.0:1787827173.621755:0:27439:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787827173.621759:0:27439:0:(file.c:2240:ll_file_io_generic()) oosfile: 2 io complete with rc: 0, result: 1024, restart: 0 00000080:00200000:3.0:1787827173.621764:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728173056 i_blocks 3365288 00000080:00200000:3.0:1787827173.621767:0:27439:0:(file.c:2326:ll_file_io_generic()) iot: 2, result: 1024 00000080:00200008:3.0:1787827173.621774:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728173056, count: 1024 00000080:00200000:3.0:1787827173.621780:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421917 of 1024 to 1024 bytes 00000080:00200000:3.0:1787827173.621787:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728173056, len 1024, copied 1024 00000080:00200000:3.0:1787827173.621823:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728174080 i_blocks 3365288 00000080:00200000:3.0:1787827173.621830:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.621839:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728174080, count: 1024 00000080:00200000:3.0:1787827173.621846:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421917 of 2048 to 1024 bytes 00000080:00200000:3.0:1787827173.621852:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728174080, len 1024, copied 1024 00000080:00200000:3.0:1787827173.621861:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728175104 i_blocks 3365288 00000080:00200000:3.0:1787827173.621868:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.621874:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728175104, count: 1024 00000080:00200000:3.0:1787827173.621881:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421917 of 3072 to 1024 bytes 00000080:00200000:3.0:1787827173.621887:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728175104, len 1024, copied 1024 00000080:00200000:3.0:1787827173.621893:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728176128 i_blocks 3365288 00000080:00200000:3.0:1787827173.621900:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.621908:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728176128, count: 1024 00000080:00200000:3.0:1787827173.621926:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421918 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.621931:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 0, original count 1024 00000080:00200000:3.0:1787827173.621936:0:27439:0:(file.c:2090:ll_file_io_generic()) oosfile: write ppos: 1728176128, bytes: 1024 00000080:00200000:3.0:1787827173.621943:0:27439:0:(file.c:1848:ll_io_set_mirror()) oosfile: desiginated mirror: 0 00000020:00200000:3.0:1787827173.621947:0:27439:0:(cl_io.c:218:cl_io_rw_init()) header@000000007788600a[0x110, 369471, [0x200000402:0x27:0x0] hash] 00000020:00200000:3.0:1787827173.621951:0:27439:0:(cl_io.c:218:cl_io_rw_init()) io range: 2 [1728176128, 1728177152) 0 0 00000080:00200000:3.0:1787827173.621955:0:27439:0:(vvp_io.c:1889:vvp_io_init()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1787827173.621965:0:27439:0:(file.c:2170:ll_file_io_generic()) Range lock [421918, 421918] 00020000:00200000:3.0:1787827173.621970:0:27439:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 412 chunk: [1728176128, 1728177152) 1728177152, 1024 00020000:00200000:3.0:1787827173.621977:0:27439:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1787827173.621986:0:27439:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [421918, 421918] 00000008:00010000:3.0:1787827173.621994:0:27439:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1787827173.621999:0:27439:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff9e35c7d81d30, osc lock ffff9e35c2ddcc30, flags 0x0 00010000:00010000:3.0:1787827173.622013:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 20/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.622028:0:27439:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (864149504 864153599) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 20/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.622043:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 24/0,2 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.622064:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 24/0,2 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.622077:0:27439:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 23/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.622093:0:27439:0:(vvp_io.c:1339:vvp_io_write_start()) oosfile: write [1728176128, 1728177152) append 0 append lockpos 0 00000080:00200000:3.0:1787827173.622099:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421918 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.622122:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728176128, len 1024, copied 1024 00000080:00200000:3.0:1787827173.622134:0:27439:0:(rw26.c:933:ll_write_end()) page@00000000c4051491[3 000000005ce0a915 2 1 00000000d6bbf043 0 0 421918] 00000080:00200000:3.0:1787827173.622139:0:27439:0:(rw26.c:933:ll_write_end()) vmpage @00000000efb48626 fffffc0001001 3:0 ffff9e34c3cb4ae0 421918 lru 00000080:00200000:3.0:1787827173.622150:0:27439:0:(rw26.c:933:ll_write_end()) osc-page@000000003ae26026 210974: 1< 0 - - > 2< 864149504 0 0 0x0 0x0 | 00000000640821b9 0000000006873caa > 3< 0 0 > 4< 0 0 8 484491264 - | - - + - > 5< - - + - | 0 - | 47 - -> 00000080:00200000:3.0:1787827173.622156:0:27439:0:(rw26.c:933:ll_write_end()) end page@00000000c4051491 00000080:00200000:3.0:1787827173.622159:0:27439:0:(rw26.c:933:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1787827173.622163:0:27439:0:(vvp_io.c:1214:vvp_io_write_commit()) commit async pages: 1, from 0, to 1024 prio 0 00000008:00010000:3.0:1787827173.622176:0:27439:0:(osc_cache.c:565:osc_extent_merge()) ### extent: 00000000e070090a ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.622194:0:27439:0:(osc_cache.c:497:__osc_extent_remove()) ### extent: 00000000e070090a ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.622204:0:27439:0:(osc_cache.c:582:osc_extent_merge()) ### extent: 000000001063fa34 ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.622214:0:27439:0:(osc_cache.c:846:osc_extent_find()) ### extent: 000000001063fa34 ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.622230:0:27439:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff9e35d0e392f8, count 1, dirtied 1 00000080:00200000:3.0:1787827173.622237:0:27439:0:(vvp_io.c:1237:vvp_io_write_commit()) Committed 1 pages 1024 bytes, tot: 1024 00000080:00200000:3.0:1787827173.622243:0:27439:0:(vvp_io.c:1421:vvp_io_write_start()) oosfile: write bytes 0, result: 1024 00000020:00010000:3.0:1787827173.622254:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_release():245 00000020:00010000:3.0:1787827173.622259:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1787827173.622267:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c7d81d30 (ffff9e35c614ce18/1) at cl_lock_cancel():142 00010000:00010000:3.0:1787827173.622277:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.622299:0:27439:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 21/0,0 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1787827173.622320:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_fini():72 00000020:00010000:3.0:1787827173.622325:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c7d81d30 (ffff9e35e04b4e18/1) at cl_lock_fini():72 00000080:00200000:3.0:1787827173.622333:0:27439:0:(vvp_io.c:533:vvp_io_advance()) advancing 1024 bytes 00000080:00200000:3.0:1787827173.622336:0:27439:0:(file.c:2207:ll_file_io_generic()) Range unlock [421918, 421918] 00000080:00200000:3.0:1787827173.622340:0:27439:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787827173.622344:0:27439:0:(file.c:2240:ll_file_io_generic()) oosfile: 2 io complete with rc: 0, result: 1024, restart: 0 00000080:00200000:3.0:1787827173.622349:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728177152 i_blocks 3365288 00000080:00200000:3.0:1787827173.622356:0:27439:0:(file.c:2326:ll_file_io_generic()) iot: 2, result: 1024 00000080:00200008:3.0:1787827173.622367:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728177152, count: 1024 00000080:00200000:3.0:1787827173.622376:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421918 of 1024 to 1024 bytes 00000080:00200000:3.0:1787827173.622382:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728177152, len 1024, copied 1024 00000080:00200000:3.0:1787827173.622391:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728178176 i_blocks 3365288 00000080:00200000:3.0:1787827173.622398:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.622405:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728178176, count: 1024 00000080:00200000:3.0:1787827173.622413:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421918 of 2048 to 1024 bytes 00000080:00200000:3.0:1787827173.622419:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728178176, len 1024, copied 1024 00000080:00200000:3.0:1787827173.622425:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728179200 i_blocks 3365288 00000080:00200000:3.0:1787827173.622430:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.622436:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728179200, count: 1024 00000080:00200000:3.0:1787827173.622443:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421918 of 3072 to 1024 bytes 00000080:00200000:3.0:1787827173.622449:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728179200, len 1024, copied 1024 00000080:00200000:3.0:1787827173.622455:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728180224 i_blocks 3365288 00000080:00200000:3.0:1787827173.622459:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.622469:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728180224, count: 1024 00000080:00200000:3.0:1787827173.622474:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421919 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.622477:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 0, original count 1024 00000080:00200000:3.0:1787827173.622481:0:27439:0:(file.c:2090:ll_file_io_generic()) oosfile: write ppos: 1728180224, bytes: 1024 00000080:00200000:3.0:1787827173.622485:0:27439:0:(file.c:1848:ll_io_set_mirror()) oosfile: desiginated mirror: 0 00000020:00200000:3.0:1787827173.622495:0:27439:0:(cl_io.c:218:cl_io_rw_init()) header@000000007788600a[0x110, 369472, [0x200000402:0x27:0x0] hash] 00000020:00200000:3.0:1787827173.622497:0:27439:0:(cl_io.c:218:cl_io_rw_init()) io range: 2 [1728180224, 1728181248) 0 0 00000080:00200000:3.0:1787827173.622504:0:27439:0:(vvp_io.c:1889:vvp_io_init()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1787827173.622514:0:27439:0:(file.c:2170:ll_file_io_generic()) Range lock [421919, 421919] 00020000:00200000:3.0:1787827173.622519:0:27439:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 412 chunk: [1728180224, 1728181248) 1728181248, 1024 00020000:00200000:3.0:1787827173.622526:0:27439:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1787827173.622534:0:27439:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [421919, 421919] 00000008:00010000:3.0:1787827173.622542:0:27439:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1787827173.622546:0:27439:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff9e35c7d81d30, osc lock ffff9e35c2ddcc30, flags 0x0 00010000:00010000:3.0:1787827173.622559:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 20/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.622574:0:27439:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (864153600 864157695) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 20/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.622596:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 24/0,2 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.622628:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 24/0,2 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.622647:0:27439:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 23/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.622657:0:27439:0:(vvp_io.c:1339:vvp_io_write_start()) oosfile: write [1728180224, 1728181248) append 0 append lockpos 0 00000080:00200000:3.0:1787827173.622664:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421919 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.622681:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728180224, len 1024, copied 1024 00000080:00200000:3.0:1787827173.622690:0:27439:0:(rw26.c:933:ll_write_end()) page@000000003b8cb944[3 000000005ce0a915 2 1 00000000d6bbf043 0 0 421919] 00000080:00200000:3.0:1787827173.622697:0:27439:0:(rw26.c:933:ll_write_end()) vmpage @000000008174201e fffffc0001001 3:0 ffff9e34c3cb4828 421919 lru 00000080:00200000:3.0:1787827173.622710:0:27439:0:(rw26.c:933:ll_write_end()) osc-page@00000000bbd7ab53 210975: 1< 0 - - > 2< 864153600 0 0 0x0 0x0 | 00000000640821b9 0000000006873caa > 3< 0 0 > 4< 0 0 8 484487168 - | - - + - > 5< - - + - | 0 - | 48 - -> 00000080:00200000:3.0:1787827173.622713:0:27439:0:(rw26.c:933:ll_write_end()) end page@000000003b8cb944 00000080:00200000:3.0:1787827173.622716:0:27439:0:(rw26.c:933:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1787827173.622719:0:27439:0:(vvp_io.c:1214:vvp_io_write_commit()) commit async pages: 1, from 0, to 1024 prio 0 00000008:00010000:3.0:1787827173.622729:0:27439:0:(osc_cache.c:565:osc_extent_merge()) ### extent: 00000000e070090a ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.622739:0:27439:0:(osc_cache.c:497:__osc_extent_remove()) ### extent: 00000000e070090a ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.622749:0:27439:0:(osc_cache.c:582:osc_extent_merge()) ### extent: 000000001063fa34 ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.622758:0:27439:0:(osc_cache.c:846:osc_extent_find()) ### extent: 000000001063fa34 ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.622767:0:27439:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff9e35d0e392f8, count 1, dirtied 1 00000080:00200000:3.0:1787827173.622769:0:27439:0:(vvp_io.c:1237:vvp_io_write_commit()) Committed 1 pages 1024 bytes, tot: 1024 00000080:00200000:3.0:1787827173.622771:0:27439:0:(vvp_io.c:1421:vvp_io_write_start()) oosfile: write bytes 0, result: 1024 00000020:00010000:3.0:1787827173.622777:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_release():245 00000020:00010000:3.0:1787827173.622782:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1787827173.622789:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c7d81d30 (ffff9e35c614ce18/1) at cl_lock_cancel():142 00010000:00010000:3.0:1787827173.622794:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.622824:0:27439:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 21/0,0 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1787827173.622838:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_fini():72 00000020:00010000:3.0:1787827173.622844:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c7d81d30 (ffff9e35e04b4e18/1) at cl_lock_fini():72 00000080:00200000:3.0:1787827173.622852:0:27439:0:(vvp_io.c:533:vvp_io_advance()) advancing 1024 bytes 00000080:00200000:3.0:1787827173.622857:0:27439:0:(file.c:2207:ll_file_io_generic()) Range unlock [421919, 421919] 00000080:00200000:3.0:1787827173.622863:0:27439:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787827173.622871:0:27439:0:(file.c:2240:ll_file_io_generic()) oosfile: 2 io complete with rc: 0, result: 1024, restart: 0 00000080:00200000:3.0:1787827173.622878:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728181248 i_blocks 3365288 00000080:00200000:3.0:1787827173.622883:0:27439:0:(file.c:2326:ll_file_io_generic()) iot: 2, result: 1024 00000080:00200008:3.0:1787827173.622891:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728181248, count: 1024 00000080:00200000:3.0:1787827173.622896:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421919 of 1024 to 1024 bytes 00000080:00200000:3.0:1787827173.622902:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728181248, len 1024, copied 1024 00000080:00200000:3.0:1787827173.622909:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728182272 i_blocks 3365288 00000080:00200000:3.0:1787827173.622913:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.622934:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728182272, count: 1024 00000080:00200000:3.0:1787827173.622940:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421919 of 2048 to 1024 bytes 00000080:00200000:3.0:1787827173.622944:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728182272, len 1024, copied 1024 00000080:00200000:3.0:1787827173.622950:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728183296 i_blocks 3365288 00000080:00200000:3.0:1787827173.622954:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.622957:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728183296, count: 1024 00000080:00200000:3.0:1787827173.622963:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421919 of 3072 to 1024 bytes 00000080:00200000:3.0:1787827173.622967:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728183296, len 1024, copied 1024 00000080:00200000:3.0:1787827173.622972:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728184320 i_blocks 3365288 00000080:00200000:3.0:1787827173.622975:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.622978:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728184320, count: 1024 00000080:00200000:3.0:1787827173.622982:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421920 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.622986:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 0, original count 1024 00000080:00200000:3.0:1787827173.622989:0:27439:0:(file.c:2090:ll_file_io_generic()) oosfile: write ppos: 1728184320, bytes: 1024 00000080:00200000:3.0:1787827173.622993:0:27439:0:(file.c:1848:ll_io_set_mirror()) oosfile: desiginated mirror: 0 00000020:00200000:3.0:1787827173.622999:0:27439:0:(cl_io.c:218:cl_io_rw_init()) header@000000007788600a[0x110, 369473, [0x200000402:0x27:0x0] hash] 00000020:00200000:3.0:1787827173.623001:0:27439:0:(cl_io.c:218:cl_io_rw_init()) io range: 2 [1728184320, 1728185344) 0 0 00000080:00200000:3.0:1787827173.623004:0:27439:0:(vvp_io.c:1889:vvp_io_init()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1787827173.623011:0:27439:0:(file.c:2170:ll_file_io_generic()) Range lock [421920, 421920] 00020000:00200000:3.0:1787827173.623015:0:27439:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 412 chunk: [1728184320, 1728185344) 1728185344, 1024 00020000:00200000:3.0:1787827173.623021:0:27439:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1787827173.623028:0:27439:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [421920, 421920] 00000008:00010000:3.0:1787827173.623033:0:27439:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1787827173.623037:0:27439:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff9e35c7d81d30, osc lock ffff9e35c2ddcc30, flags 0x0 00010000:00010000:3.0:1787827173.623046:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 20/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.623060:0:27439:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (864157696 864161791) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 20/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.623073:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 24/0,2 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.623097:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 24/0,2 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.623108:0:27439:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 23/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.623114:0:27439:0:(vvp_io.c:1339:vvp_io_write_start()) oosfile: write [1728184320, 1728185344) append 0 append lockpos 0 00000080:00200000:3.0:1787827173.623121:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421920 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.623137:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728184320, len 1024, copied 1024 00000080:00200000:3.0:1787827173.623144:0:27439:0:(rw26.c:933:ll_write_end()) page@00000000d5022d8d[3 000000005ce0a915 2 1 00000000d6bbf043 0 0 421920] 00000080:00200000:3.0:1787827173.623149:0:27439:0:(rw26.c:933:ll_write_end()) vmpage @000000007283673f fffffc0001001 3:0 ffff9e34c3cb4570 421920 lru 00000080:00200000:3.0:1787827173.623156:0:27439:0:(rw26.c:933:ll_write_end()) osc-page@00000000ec5ee7ad 210976: 1< 0 - - > 2< 864157696 0 0 0x0 0x0 | 00000000640821b9 0000000006873caa > 3< 0 0 > 4< 0 0 8 484483072 - | - - + - > 5< - - + - | 0 - | 49 - -> 00000080:00200000:3.0:1787827173.623158:0:27439:0:(rw26.c:933:ll_write_end()) end page@00000000d5022d8d 00000080:00200000:3.0:1787827173.623159:0:27439:0:(rw26.c:933:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1787827173.623163:0:27439:0:(vvp_io.c:1214:vvp_io_write_commit()) commit async pages: 1, from 0, to 1024 prio 0 00000008:00010000:3.0:1787827173.623170:0:27439:0:(osc_cache.c:565:osc_extent_merge()) ### extent: 00000000e070090a ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.623179:0:27439:0:(osc_cache.c:497:__osc_extent_remove()) ### extent: 00000000e070090a ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.623188:0:27439:0:(osc_cache.c:582:osc_extent_merge()) ### extent: 000000001063fa34 ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.623194:0:27439:0:(osc_cache.c:846:osc_extent_find()) ### extent: 000000001063fa34 ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.623217:0:27439:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff9e35d0e392f8, count 1, dirtied 1 00000080:00200000:3.0:1787827173.623222:0:27439:0:(vvp_io.c:1237:vvp_io_write_commit()) Committed 1 pages 1024 bytes, tot: 1024 00000080:00200000:3.0:1787827173.623227:0:27439:0:(vvp_io.c:1421:vvp_io_write_start()) oosfile: write bytes 0, result: 1024 00000020:00010000:3.0:1787827173.623237:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_release():245 00000020:00010000:3.0:1787827173.623244:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1787827173.623249:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c7d81d30 (ffff9e35c614ce18/1) at cl_lock_cancel():142 00010000:00010000:3.0:1787827173.623260:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.623284:0:27439:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 21/0,0 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1787827173.623297:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_fini():72 00000020:00010000:3.0:1787827173.623301:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c7d81d30 (ffff9e35e04b4e18/1) at cl_lock_fini():72 00000080:00200000:3.0:1787827173.623311:0:27439:0:(vvp_io.c:533:vvp_io_advance()) advancing 1024 bytes 00000080:00200000:3.0:1787827173.623317:0:27439:0:(file.c:2207:ll_file_io_generic()) Range unlock [421920, 421920] 00000080:00200000:3.0:1787827173.623325:0:27439:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787827173.623332:0:27439:0:(file.c:2240:ll_file_io_generic()) oosfile: 2 io complete with rc: 0, result: 1024, restart: 0 00000080:00200000:3.0:1787827173.623337:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728185344 i_blocks 3365288 00000080:00200000:3.0:1787827173.623339:0:27439:0:(file.c:2326:ll_file_io_generic()) iot: 2, result: 1024 00000080:00200008:3.0:1787827173.623347:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728185344, count: 1024 00000080:00200000:3.0:1787827173.623352:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421920 of 1024 to 1024 bytes 00000080:00200000:3.0:1787827173.623356:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728185344, len 1024, copied 1024 00000080:00200000:3.0:1787827173.623362:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728186368 i_blocks 3365288 00000080:00200000:3.0:1787827173.623366:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.623372:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728186368, count: 1024 00000080:00200000:3.0:1787827173.623378:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421920 of 2048 to 1024 bytes 00000080:00200000:3.0:1787827173.623384:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728186368, len 1024, copied 1024 00000080:00200000:3.0:1787827173.623393:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728187392 i_blocks 3365288 00000080:00200000:3.0:1787827173.623399:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.623406:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728187392, count: 1024 00000080:00200000:3.0:1787827173.623576:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421920 of 3072 to 1024 bytes 00000080:00200000:3.0:1787827173.623581:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728187392, len 1024, copied 1024 00000080:00200000:3.0:1787827173.623593:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728188416 i_blocks 3365288 00000080:00200000:3.0:1787827173.623600:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.623609:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728188416, count: 1024 00000080:00200000:3.0:1787827173.623658:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421921 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.623664:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 0, original count 1024 00000080:00200000:3.0:1787827173.623669:0:27439:0:(file.c:2090:ll_file_io_generic()) oosfile: write ppos: 1728188416, bytes: 1024 00000080:00200000:3.0:1787827173.623672:0:27439:0:(file.c:1848:ll_io_set_mirror()) oosfile: desiginated mirror: 0 00000020:00200000:3.0:1787827173.623682:0:27439:0:(cl_io.c:218:cl_io_rw_init()) header@000000007788600a[0x110, 369474, [0x200000402:0x27:0x0] hash] 00000020:00200000:3.0:1787827173.623685:0:27439:0:(cl_io.c:218:cl_io_rw_init()) io range: 2 [1728188416, 1728189440) 0 0 00000080:00200000:3.0:1787827173.623690:0:27439:0:(vvp_io.c:1889:vvp_io_init()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1787827173.623702:0:27439:0:(file.c:2170:ll_file_io_generic()) Range lock [421921, 421921] 00020000:00200000:3.0:1787827173.623709:0:27439:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 412 chunk: [1728188416, 1728189440) 1728189440, 1024 00020000:00200000:3.0:1787827173.623717:0:27439:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1787827173.623726:0:27439:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [421921, 421921] 00000008:00010000:3.0:1787827173.623734:0:27439:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1787827173.623739:0:27439:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff9e35c7d81d30, osc lock ffff9e35c2ddcc30, flags 0x0 00010000:00010000:3.0:1787827173.623752:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 20/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.623771:0:27439:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (864161792 864165887) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 20/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.623818:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 24/0,2 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.623857:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 24/0,2 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.623876:0:27439:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 23/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.623896:0:27439:0:(vvp_io.c:1339:vvp_io_write_start()) oosfile: write [1728188416, 1728189440) append 0 append lockpos 0 00000080:00200000:3.0:1787827173.623906:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421921 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.623944:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728188416, len 1024, copied 1024 00000080:00200000:3.0:1787827173.623956:0:27439:0:(rw26.c:933:ll_write_end()) page@0000000052b59519[3 000000005ce0a915 2 1 00000000d6bbf043 0 0 421921] 00000080:00200000:3.0:1787827173.623961:0:27439:0:(rw26.c:933:ll_write_end()) vmpage @00000000df15480e fffffc0001001 3:0 ffff9e34c3cb4e80 421921 lru 00000080:00200000:3.0:1787827173.623979:0:27439:0:(rw26.c:933:ll_write_end()) osc-page@000000001b6fc5d6 210977: 1< 0 - - > 2< 864161792 0 0 0x0 0x0 | 00000000640821b9 0000000006873caa > 3< 0 0 > 4< 0 0 8 484478976 - | - - + - > 5< - - + - | 0 - | 50 - -> 00000080:00200000:3.0:1787827173.623984:0:27439:0:(rw26.c:933:ll_write_end()) end page@0000000052b59519 00000080:00200000:3.0:1787827173.623988:0:27439:0:(rw26.c:933:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1787827173.623992:0:27439:0:(vvp_io.c:1214:vvp_io_write_commit()) commit async pages: 1, from 0, to 1024 prio 0 00000008:00010000:3.0:1787827173.624004:0:27439:0:(osc_cache.c:565:osc_extent_merge()) ### extent: 00000000e070090a ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.624028:0:27439:0:(osc_cache.c:497:__osc_extent_remove()) ### extent: 00000000e070090a ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.624051:0:27439:0:(osc_cache.c:582:osc_extent_merge()) ### extent: 000000001063fa34 ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.624065:0:27439:0:(osc_cache.c:846:osc_extent_find()) ### extent: 000000001063fa34 ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.624083:0:27439:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff9e35d0e392f8, count 1, dirtied 1 00000080:00200000:3.0:1787827173.624089:0:27439:0:(vvp_io.c:1237:vvp_io_write_commit()) Committed 1 pages 1024 bytes, tot: 1024 00000080:00200000:3.0:1787827173.624095:0:27439:0:(vvp_io.c:1421:vvp_io_write_start()) oosfile: write bytes 0, result: 1024 00000020:00010000:3.0:1787827173.624106:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_release():245 00000020:00010000:3.0:1787827173.624112:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1787827173.624119:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c7d81d30 (ffff9e35c614ce18/1) at cl_lock_cancel():142 00010000:00010000:3.0:1787827173.624128:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.624152:0:27439:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 21/0,0 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1787827173.624180:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_fini():72 00000020:00010000:3.0:1787827173.624188:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c7d81d30 (ffff9e35e04b4e18/1) at cl_lock_fini():72 00000080:00200000:3.0:1787827173.624197:0:27439:0:(vvp_io.c:533:vvp_io_advance()) advancing 1024 bytes 00000080:00200000:3.0:1787827173.624202:0:27439:0:(file.c:2207:ll_file_io_generic()) Range unlock [421921, 421921] 00000080:00200000:3.0:1787827173.624211:0:27439:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787827173.624217:0:27439:0:(file.c:2240:ll_file_io_generic()) oosfile: 2 io complete with rc: 0, result: 1024, restart: 0 00000080:00200000:3.0:1787827173.624226:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728189440 i_blocks 3373480 00000080:00200000:3.0:1787827173.624233:0:27439:0:(file.c:2326:ll_file_io_generic()) iot: 2, result: 1024 00000080:00200008:3.0:1787827173.624244:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728189440, count: 1024 00000080:00200000:3.0:1787827173.624255:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421921 of 1024 to 1024 bytes 00000080:00200000:3.0:1787827173.624260:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728189440, len 1024, copied 1024 00000080:00200000:3.0:1787827173.624270:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728190464 i_blocks 3373480 00000080:00200000:3.0:1787827173.624276:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.624284:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728190464, count: 1024 00000080:00200000:3.0:1787827173.624291:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421921 of 2048 to 1024 bytes 00000080:00200000:3.0:1787827173.624297:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728190464, len 1024, copied 1024 00000080:00200000:3.0:1787827173.624305:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728191488 i_blocks 3373480 00000080:00200000:3.0:1787827173.624313:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.624320:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728191488, count: 1024 00000080:00200000:3.0:1787827173.624328:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421921 of 3072 to 1024 bytes 00000080:00200000:3.0:1787827173.624332:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728191488, len 1024, copied 1024 00000080:00200000:3.0:1787827173.624340:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728192512 i_blocks 3373480 00000080:00200000:3.0:1787827173.624346:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.624354:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728192512, count: 1024 00000080:00200000:3.0:1787827173.624361:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421922 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.624367:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 0, original count 1024 00000080:00200000:3.0:1787827173.624372:0:27439:0:(file.c:2090:ll_file_io_generic()) oosfile: write ppos: 1728192512, bytes: 1024 00000080:00200000:3.0:1787827173.624379:0:27439:0:(file.c:1848:ll_io_set_mirror()) oosfile: desiginated mirror: 0 00000020:00200000:3.0:1787827173.624387:0:27439:0:(cl_io.c:218:cl_io_rw_init()) header@000000007788600a[0x110, 369475, [0x200000402:0x27:0x0] hash] 00000020:00200000:3.0:1787827173.624391:0:27439:0:(cl_io.c:218:cl_io_rw_init()) io range: 2 [1728192512, 1728193536) 0 0 00000080:00200000:3.0:1787827173.624396:0:27439:0:(vvp_io.c:1889:vvp_io_init()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1787827173.624409:0:27439:0:(file.c:2170:ll_file_io_generic()) Range lock [421922, 421922] 00020000:00200000:3.0:1787827173.624415:0:27439:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 412 chunk: [1728192512, 1728193536) 1728193536, 1024 00020000:00200000:3.0:1787827173.624421:0:27439:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1787827173.624429:0:27439:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [421922, 421922] 00000008:00010000:3.0:1787827173.624439:0:27439:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1787827173.624443:0:27439:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff9e35c7d81d30, osc lock ffff9e35c2ddcc30, flags 0x0 00010000:00010000:3.0:1787827173.624458:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 20/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.624478:0:27439:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (864165888 864169983) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 20/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.624495:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 24/0,2 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.624531:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 24/0,2 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.624558:0:27439:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 23/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.624580:0:27439:0:(vvp_io.c:1339:vvp_io_write_start()) oosfile: write [1728192512, 1728193536) append 0 append lockpos 0 00000080:00200000:3.0:1787827173.624592:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421922 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.624613:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728192512, len 1024, copied 1024 00000080:00200000:3.0:1787827173.624625:0:27439:0:(rw26.c:933:ll_write_end()) page@0000000050f6d00c[3 000000005ce0a915 2 1 00000000d6bbf043 0 0 421922] 00000080:00200000:3.0:1787827173.624631:0:27439:0:(rw26.c:933:ll_write_end()) vmpage @00000000584f95cf fffffc0001001 3:0 ffff9e34c3cb4740 421922 lru 00000080:00200000:3.0:1787827173.624643:0:27439:0:(rw26.c:933:ll_write_end()) osc-page@00000000ef870ad3 210978: 1< 0 - - > 2< 864165888 0 0 0x0 0x0 | 00000000640821b9 0000000006873caa > 3< 0 0 > 4< 0 0 8 484474880 - | - - + - > 5< - - + - | 0 - | 51 - -> 00000080:00200000:3.0:1787827173.624647:0:27439:0:(rw26.c:933:ll_write_end()) end page@0000000050f6d00c 00000080:00200000:3.0:1787827173.624648:0:27439:0:(rw26.c:933:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1787827173.624651:0:27439:0:(vvp_io.c:1214:vvp_io_write_commit()) commit async pages: 1, from 0, to 1024 prio 0 00000008:00010000:3.0:1787827173.624663:0:27439:0:(osc_cache.c:565:osc_extent_merge()) ### extent: 00000000e070090a ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.624683:0:27439:0:(osc_cache.c:497:__osc_extent_remove()) ### extent: 00000000e070090a ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.624702:0:27439:0:(osc_cache.c:582:osc_extent_merge()) ### extent: 000000001063fa34 ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.624714:0:27439:0:(osc_cache.c:846:osc_extent_find()) ### extent: 000000001063fa34 ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.624743:0:27439:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff9e35d0e392f8, count 1, dirtied 1 00000080:00200000:3.0:1787827173.624749:0:27439:0:(vvp_io.c:1237:vvp_io_write_commit()) Committed 1 pages 1024 bytes, tot: 1024 00000080:00200000:3.0:1787827173.624754:0:27439:0:(vvp_io.c:1421:vvp_io_write_start()) oosfile: write bytes 0, result: 1024 00000020:00010000:3.0:1787827173.624765:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_release():245 00000020:00010000:3.0:1787827173.624773:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1787827173.624778:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c7d81d30 (ffff9e35c614ce18/1) at cl_lock_cancel():142 00010000:00010000:3.0:1787827173.624784:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.624830:0:27439:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 21/0,0 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1787827173.624853:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_fini():72 00000020:00010000:3.0:1787827173.624861:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c7d81d30 (ffff9e35e04b4e18/1) at cl_lock_fini():72 00000080:00200000:3.0:1787827173.624867:0:27439:0:(vvp_io.c:533:vvp_io_advance()) advancing 1024 bytes 00000080:00200000:3.0:1787827173.624873:0:27439:0:(file.c:2207:ll_file_io_generic()) Range unlock [421922, 421922] 00000080:00200000:3.0:1787827173.624879:0:27439:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787827173.624887:0:27439:0:(file.c:2240:ll_file_io_generic()) oosfile: 2 io complete with rc: 0, result: 1024, restart: 0 00000080:00200000:3.0:1787827173.624896:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728193536 i_blocks 3373480 00000080:00200000:3.0:1787827173.624901:0:27439:0:(file.c:2326:ll_file_io_generic()) iot: 2, result: 1024 00000080:00200008:3.0:1787827173.624912:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728193536, count: 1024 00000080:00200000:3.0:1787827173.624933:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421922 of 1024 to 1024 bytes 00000080:00200000:3.0:1787827173.624939:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728193536, len 1024, copied 1024 00000080:00200000:3.0:1787827173.624948:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728194560 i_blocks 3373480 00000080:00200000:3.0:1787827173.624953:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.624962:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728194560, count: 1024 00000080:00200000:3.0:1787827173.624969:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421922 of 2048 to 1024 bytes 00000080:00200000:3.0:1787827173.624976:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728194560, len 1024, copied 1024 00000080:00200000:3.0:1787827173.624984:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728195584 i_blocks 3373480 00000080:00200000:3.0:1787827173.624990:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.624999:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728195584, count: 1024 00000080:00200000:3.0:1787827173.625006:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421922 of 3072 to 1024 bytes 00000080:00200000:3.0:1787827173.625012:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728195584, len 1024, copied 1024 00000080:00200000:3.0:1787827173.625020:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728196608 i_blocks 3373480 00000080:00200000:3.0:1787827173.625026:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.625035:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728196608, count: 1024 00000080:00200000:3.0:1787827173.625044:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421923 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.625049:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 0, original count 1024 00000080:00200000:3.0:1787827173.625053:0:27439:0:(file.c:2090:ll_file_io_generic()) oosfile: write ppos: 1728196608, bytes: 1024 00000080:00200000:3.0:1787827173.625059:0:27439:0:(file.c:1848:ll_io_set_mirror()) oosfile: desiginated mirror: 0 00000020:00200000:3.0:1787827173.625067:0:27439:0:(cl_io.c:218:cl_io_rw_init()) header@000000007788600a[0x110, 369476, [0x200000402:0x27:0x0] hash] 00000020:00200000:3.0:1787827173.625070:0:27439:0:(cl_io.c:218:cl_io_rw_init()) io range: 2 [1728196608, 1728197632) 0 0 00000080:00200000:3.0:1787827173.625076:0:27439:0:(vvp_io.c:1889:vvp_io_init()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1787827173.625088:0:27439:0:(file.c:2170:ll_file_io_generic()) Range lock [421923, 421923] 00020000:00200000:3.0:1787827173.625094:0:27439:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 412 chunk: [1728196608, 1728197632) 1728197632, 1024 00020000:00200000:3.0:1787827173.625099:0:27439:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1787827173.625110:0:27439:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [421923, 421923] 00000008:00010000:3.0:1787827173.625119:0:27439:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1787827173.625124:0:27439:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff9e35c7d81d30, osc lock ffff9e35c2ddcc30, flags 0x0 00010000:00010000:3.0:1787827173.625137:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 20/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.625150:0:27439:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (864169984 864174079) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 20/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.625173:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 24/0,2 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.625214:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 24/0,2 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.625231:0:27439:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 23/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.625251:0:27439:0:(vvp_io.c:1339:vvp_io_write_start()) oosfile: write [1728196608, 1728197632) append 0 append lockpos 0 00000080:00200000:3.0:1787827173.625262:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421923 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.625283:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728196608, len 1024, copied 1024 00000080:00200000:3.0:1787827173.625291:0:27439:0:(rw26.c:933:ll_write_end()) page@00000000387437a0[3 000000005ce0a915 2 1 00000000d6bbf043 0 0 421923] 00000080:00200000:3.0:1787827173.625300:0:27439:0:(rw26.c:933:ll_write_end()) vmpage @000000009b97465a fffffc0001001 3:0 ffff9e34c3cb4910 421923 lru 00000080:00200000:3.0:1787827173.625318:0:27439:0:(rw26.c:933:ll_write_end()) osc-page@000000006185f958 210979: 1< 0 - - > 2< 864169984 0 0 0x0 0x0 | 00000000640821b9 0000000006873caa > 3< 0 0 > 4< 0 0 8 484470784 - | - - + - > 5< - - + - | 0 - | 52 - -> 00000080:00200000:3.0:1787827173.625323:0:27439:0:(rw26.c:933:ll_write_end()) end page@00000000387437a0 00000080:00200000:3.0:1787827173.625327:0:27439:0:(rw26.c:933:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1787827173.625331:0:27439:0:(vvp_io.c:1214:vvp_io_write_commit()) commit async pages: 1, from 0, to 1024 prio 0 00000008:00010000:3.0:1787827173.625347:0:27439:0:(osc_cache.c:565:osc_extent_merge()) ### extent: 00000000e070090a ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.625373:0:27439:0:(osc_cache.c:497:__osc_extent_remove()) ### extent: 00000000e070090a ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.625391:0:27439:0:(osc_cache.c:582:osc_extent_merge()) ### extent: 000000001063fa34 ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.625412:0:27439:0:(osc_cache.c:846:osc_extent_find()) ### extent: 000000001063fa34 ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.625432:0:27439:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff9e35d0e392f8, count 1, dirtied 1 00000080:00200000:3.0:1787827173.625439:0:27439:0:(vvp_io.c:1237:vvp_io_write_commit()) Committed 1 pages 1024 bytes, tot: 1024 00000080:00200000:3.0:1787827173.625444:0:27439:0:(vvp_io.c:1421:vvp_io_write_start()) oosfile: write bytes 0, result: 1024 00000020:00010000:3.0:1787827173.625453:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_release():245 00000020:00010000:3.0:1787827173.625461:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1787827173.625467:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c7d81d30 (ffff9e35c614ce18/1) at cl_lock_cancel():142 00010000:00010000:3.0:1787827173.625476:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.625496:0:27439:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 21/0,0 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1787827173.625518:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_fini():72 00000020:00010000:3.0:1787827173.625525:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c7d81d30 (ffff9e35e04b4e18/1) at cl_lock_fini():72 00000080:00200000:3.0:1787827173.625533:0:27439:0:(vvp_io.c:533:vvp_io_advance()) advancing 1024 bytes 00000080:00200000:3.0:1787827173.625539:0:27439:0:(file.c:2207:ll_file_io_generic()) Range unlock [421923, 421923] 00000080:00200000:3.0:1787827173.625547:0:27439:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787827173.625552:0:27439:0:(file.c:2240:ll_file_io_generic()) oosfile: 2 io complete with rc: 0, result: 1024, restart: 0 00000080:00200000:3.0:1787827173.625559:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728197632 i_blocks 3373480 00000080:00200000:3.0:1787827173.625564:0:27439:0:(file.c:2326:ll_file_io_generic()) iot: 2, result: 1024 00000080:00200008:3.0:1787827173.625576:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728197632, count: 1024 00000080:00200000:3.0:1787827173.625585:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421923 of 1024 to 1024 bytes 00000080:00200000:3.0:1787827173.625591:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728197632, len 1024, copied 1024 00000080:00200000:3.0:1787827173.625602:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728198656 i_blocks 3373480 00000080:00200000:3.0:1787827173.625607:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.625616:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728198656, count: 1024 00000080:00200000:3.0:1787827173.625625:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421923 of 2048 to 1024 bytes 00000080:00200000:3.0:1787827173.625631:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728198656, len 1024, copied 1024 00000080:00200000:3.0:1787827173.625640:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728199680 i_blocks 3373480 00000080:00200000:3.0:1787827173.625645:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.625654:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728199680, count: 1024 00000080:00200000:3.0:1787827173.625661:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421923 of 3072 to 1024 bytes 00000080:00200000:3.0:1787827173.625667:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728199680, len 1024, copied 1024 00000080:00200000:3.0:1787827173.625678:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728200704 i_blocks 3373480 00000080:00200000:3.0:1787827173.625683:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.625693:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728200704, count: 1024 00000080:00200000:3.0:1787827173.625702:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421924 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.625707:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 0, original count 1024 00000080:00200000:3.0:1787827173.625711:0:27439:0:(file.c:2090:ll_file_io_generic()) oosfile: write ppos: 1728200704, bytes: 1024 00000080:00200000:3.0:1787827173.625718:0:27439:0:(file.c:1848:ll_io_set_mirror()) oosfile: desiginated mirror: 0 00000020:00200000:3.0:1787827173.625725:0:27439:0:(cl_io.c:218:cl_io_rw_init()) header@000000007788600a[0x110, 369477, [0x200000402:0x27:0x0] hash] 00000020:00200000:3.0:1787827173.625727:0:27439:0:(cl_io.c:218:cl_io_rw_init()) io range: 2 [1728200704, 1728201728) 0 0 00000080:00200000:3.0:1787827173.625732:0:27439:0:(vvp_io.c:1889:vvp_io_init()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1787827173.625741:0:27439:0:(file.c:2170:ll_file_io_generic()) Range lock [421924, 421924] 00020000:00200000:3.0:1787827173.625747:0:27439:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 412 chunk: [1728200704, 1728201728) 1728201728, 1024 00020000:00200000:3.0:1787827173.625750:0:27439:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1787827173.625758:0:27439:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [421924, 421924] 00000008:00010000:3.0:1787827173.625768:0:27439:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1787827173.625772:0:27439:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff9e35c7d81d30, osc lock ffff9e35c2ddcc30, flags 0x0 00010000:00010000:3.0:1787827173.625787:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 20/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.625828:0:27439:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (864174080 864178175) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 20/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.625849:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 24/0,2 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.625873:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 24/0,2 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.625881:0:27439:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 23/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.625903:0:27439:0:(vvp_io.c:1339:vvp_io_write_start()) oosfile: write [1728200704, 1728201728) append 0 append lockpos 0 00000080:00200000:3.0:1787827173.625925:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421924 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.625950:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728200704, len 1024, copied 1024 00000080:00200000:3.0:1787827173.625961:0:27439:0:(rw26.c:933:ll_write_end()) page@000000003bebed95[3 000000005ce0a915 2 1 00000000d6bbf043 0 0 421924] 00000080:00200000:3.0:1787827173.625967:0:27439:0:(rw26.c:933:ll_write_end()) vmpage @00000000a2777576 fffffc0001001 3:0 ffff9e34c3cb40e8 421924 lru 00000080:00200000:3.0:1787827173.625985:0:27439:0:(rw26.c:933:ll_write_end()) osc-page@00000000ca328609 210980: 1< 0 - - > 2< 864174080 0 0 0x0 0x0 | 00000000640821b9 0000000006873caa > 3< 0 0 > 4< 0 0 8 484466688 - | - - + - > 5< - - + - | 0 - | 53 - -> 00000080:00200000:3.0:1787827173.625989:0:27439:0:(rw26.c:933:ll_write_end()) end page@000000003bebed95 00000080:00200000:3.0:1787827173.625991:0:27439:0:(rw26.c:933:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1787827173.625994:0:27439:0:(vvp_io.c:1214:vvp_io_write_commit()) commit async pages: 1, from 0, to 1024 prio 0 00000008:00010000:3.0:1787827173.626005:0:27439:0:(osc_cache.c:565:osc_extent_merge()) ### extent: 00000000e070090a ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.626023:0:27439:0:(osc_cache.c:497:__osc_extent_remove()) ### extent: 00000000e070090a ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.626038:0:27439:0:(osc_cache.c:582:osc_extent_merge()) ### extent: 000000001063fa34 ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.626057:0:27439:0:(osc_cache.c:846:osc_extent_find()) ### extent: 000000001063fa34 ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.626082:0:27439:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff9e35d0e392f8, count 1, dirtied 1 00000080:00200000:3.0:1787827173.626087:0:27439:0:(vvp_io.c:1237:vvp_io_write_commit()) Committed 1 pages 1024 bytes, tot: 1024 00000080:00200000:3.0:1787827173.626093:0:27439:0:(vvp_io.c:1421:vvp_io_write_start()) oosfile: write bytes 0, result: 1024 00000020:00010000:3.0:1787827173.626103:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_release():245 00000020:00010000:3.0:1787827173.626108:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1787827173.626112:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c7d81d30 (ffff9e35c614ce18/1) at cl_lock_cancel():142 00010000:00010000:3.0:1787827173.626118:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.626135:0:27439:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 21/0,0 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1787827173.626152:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_fini():72 00000020:00010000:3.0:1787827173.626159:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c7d81d30 (ffff9e35e04b4e18/1) at cl_lock_fini():72 00000080:00200000:3.0:1787827173.626166:0:27439:0:(vvp_io.c:533:vvp_io_advance()) advancing 1024 bytes 00000080:00200000:3.0:1787827173.626172:0:27439:0:(file.c:2207:ll_file_io_generic()) Range unlock [421924, 421924] 00000080:00200000:3.0:1787827173.626179:0:27439:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787827173.626186:0:27439:0:(file.c:2240:ll_file_io_generic()) oosfile: 2 io complete with rc: 0, result: 1024, restart: 0 00000080:00200000:3.0:1787827173.626195:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728201728 i_blocks 3373480 00000080:00200000:3.0:1787827173.626200:0:27439:0:(file.c:2326:ll_file_io_generic()) iot: 2, result: 1024 00000080:00200008:3.0:1787827173.626212:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728201728, count: 1024 00000080:00200000:3.0:1787827173.626223:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421924 of 1024 to 1024 bytes 00000080:00200000:3.0:1787827173.626229:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728201728, len 1024, copied 1024 00000080:00200000:3.0:1787827173.626240:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728202752 i_blocks 3373480 00000080:00200000:3.0:1787827173.626246:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.626254:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728202752, count: 1024 00000080:00200000:3.0:1787827173.626262:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421924 of 2048 to 1024 bytes 00000080:00200000:3.0:1787827173.626268:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728202752, len 1024, copied 1024 00000080:00200000:3.0:1787827173.626275:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728203776 i_blocks 3373480 00000080:00200000:3.0:1787827173.626282:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.626289:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728203776, count: 1024 00000080:00200000:3.0:1787827173.626298:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421924 of 3072 to 1024 bytes 00000080:00200000:3.0:1787827173.626304:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728203776, len 1024, copied 1024 00000080:00200000:3.0:1787827173.626311:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728204800 i_blocks 3373480 00000080:00200000:3.0:1787827173.626318:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.626325:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728204800, count: 1024 00000080:00200000:3.0:1787827173.626333:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421925 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.626337:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 0, original count 1024 00000080:00200000:3.0:1787827173.626342:0:27439:0:(file.c:2090:ll_file_io_generic()) oosfile: write ppos: 1728204800, bytes: 1024 00000080:00200000:3.0:1787827173.626348:0:27439:0:(file.c:1848:ll_io_set_mirror()) oosfile: desiginated mirror: 0 00000020:00200000:3.0:1787827173.626356:0:27439:0:(cl_io.c:218:cl_io_rw_init()) header@000000007788600a[0x110, 369478, [0x200000402:0x27:0x0] hash] 00000020:00200000:3.0:1787827173.626360:0:27439:0:(cl_io.c:218:cl_io_rw_init()) io range: 2 [1728204800, 1728205824) 0 0 00000080:00200000:3.0:1787827173.626366:0:27439:0:(vvp_io.c:1889:vvp_io_init()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1787827173.626375:0:27439:0:(file.c:2170:ll_file_io_generic()) Range lock [421925, 421925] 00020000:00200000:3.0:1787827173.626380:0:27439:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 412 chunk: [1728204800, 1728205824) 1728205824, 1024 00020000:00200000:3.0:1787827173.626386:0:27439:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1787827173.626393:0:27439:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [421925, 421925] 00000008:00010000:3.0:1787827173.626401:0:27439:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1787827173.626404:0:27439:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff9e35c7d81d30, osc lock ffff9e35c2ddcc30, flags 0x0 00010000:00010000:3.0:1787827173.626413:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 20/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.626424:0:27439:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (864178176 864182271) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 20/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.626447:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 24/0,2 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.626471:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 24/0,2 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.626488:0:27439:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 23/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.626506:0:27439:0:(vvp_io.c:1339:vvp_io_write_start()) oosfile: write [1728204800, 1728205824) append 0 append lockpos 0 00000080:00200000:3.0:1787827173.626514:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421925 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.626537:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728204800, len 1024, copied 1024 00000080:00200000:3.0:1787827173.626548:0:27439:0:(rw26.c:933:ll_write_end()) page@000000003dc4e106[3 000000005ce0a915 2 1 00000000d6bbf043 0 0 421925] 00000080:00200000:3.0:1787827173.626555:0:27439:0:(rw26.c:933:ll_write_end()) vmpage @0000000020b9cc7f fffffc0001001 3:0 ffff9e34c3cb4cb0 421925 lru 00000080:00200000:3.0:1787827173.626571:0:27439:0:(rw26.c:933:ll_write_end()) osc-page@00000000f4b48c06 210981: 1< 0 - - > 2< 864178176 0 0 0x0 0x0 | 00000000640821b9 0000000006873caa > 3< 0 0 > 4< 0 0 8 484462592 - | - - + - > 5< - - + - | 0 - | 54 - -> 00000080:00200000:3.0:1787827173.626576:0:27439:0:(rw26.c:933:ll_write_end()) end page@000000003dc4e106 00000080:00200000:3.0:1787827173.626578:0:27439:0:(rw26.c:933:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1787827173.626581:0:27439:0:(vvp_io.c:1214:vvp_io_write_commit()) commit async pages: 1, from 0, to 1024 prio 0 00000008:00010000:3.0:1787827173.626596:0:27439:0:(osc_cache.c:565:osc_extent_merge()) ### extent: 00000000e070090a ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.626618:0:27439:0:(osc_cache.c:497:__osc_extent_remove()) ### extent: 00000000e070090a ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.626633:0:27439:0:(osc_cache.c:582:osc_extent_merge()) ### extent: 000000001063fa34 ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.626646:0:27439:0:(osc_cache.c:846:osc_extent_find()) ### extent: 000000001063fa34 ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.626665:0:27439:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff9e35d0e392f8, count 1, dirtied 1 00000080:00200000:3.0:1787827173.626670:0:27439:0:(vvp_io.c:1237:vvp_io_write_commit()) Committed 1 pages 1024 bytes, tot: 1024 00000080:00200000:3.0:1787827173.626672:0:27439:0:(vvp_io.c:1421:vvp_io_write_start()) oosfile: write bytes 0, result: 1024 00000020:00010000:3.0:1787827173.626679:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_release():245 00000020:00010000:3.0:1787827173.626684:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1787827173.626690:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c7d81d30 (ffff9e35c614ce18/1) at cl_lock_cancel():142 00010000:00010000:3.0:1787827173.626696:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.626711:0:27439:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 21/0,0 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1787827173.626729:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_fini():72 00000020:00010000:3.0:1787827173.626734:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c7d81d30 (ffff9e35e04b4e18/1) at cl_lock_fini():72 00000080:00200000:3.0:1787827173.626744:0:27439:0:(vvp_io.c:533:vvp_io_advance()) advancing 1024 bytes 00000080:00200000:3.0:1787827173.626750:0:27439:0:(file.c:2207:ll_file_io_generic()) Range unlock [421925, 421925] 00000080:00200000:3.0:1787827173.626757:0:27439:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787827173.626763:0:27439:0:(file.c:2240:ll_file_io_generic()) oosfile: 2 io complete with rc: 0, result: 1024, restart: 0 00000080:00200000:3.0:1787827173.626771:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728205824 i_blocks 3373480 00000080:00200000:3.0:1787827173.626777:0:27439:0:(file.c:2326:ll_file_io_generic()) iot: 2, result: 1024 00000080:00200008:3.0:1787827173.626789:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728205824, count: 1024 00000080:00200000:3.0:1787827173.626818:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421925 of 1024 to 1024 bytes 00000080:00200000:3.0:1787827173.626823:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728205824, len 1024, copied 1024 00000080:00200000:3.0:1787827173.626832:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728206848 i_blocks 3373480 00000080:00200000:3.0:1787827173.626837:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.626846:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728206848, count: 1024 00000080:00200000:3.0:1787827173.626852:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421925 of 2048 to 1024 bytes 00000080:00200000:3.0:1787827173.626857:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728206848, len 1024, copied 1024 00000080:00200000:3.0:1787827173.626862:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728207872 i_blocks 3373480 00000080:00200000:3.0:1787827173.626867:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.626873:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728207872, count: 1024 00000080:00200000:3.0:1787827173.626877:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421925 of 3072 to 1024 bytes 00000080:00200000:3.0:1787827173.626881:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728207872, len 1024, copied 1024 00000080:00200000:3.0:1787827173.626889:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728208896 i_blocks 3373480 00000080:00200000:3.0:1787827173.626894:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.626901:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728208896, count: 1024 00000080:00200000:3.0:1787827173.626908:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421926 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.626926:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 0, original count 1024 00000080:00200000:3.0:1787827173.626931:0:27439:0:(file.c:2090:ll_file_io_generic()) oosfile: write ppos: 1728208896, bytes: 1024 00000080:00200000:3.0:1787827173.626935:0:27439:0:(file.c:1848:ll_io_set_mirror()) oosfile: desiginated mirror: 0 00000020:00200000:3.0:1787827173.626945:0:27439:0:(cl_io.c:218:cl_io_rw_init()) header@000000007788600a[0x110, 369479, [0x200000402:0x27:0x0] hash] 00000020:00200000:3.0:1787827173.626948:0:27439:0:(cl_io.c:218:cl_io_rw_init()) io range: 2 [1728208896, 1728209920) 0 0 00000080:00200000:3.0:1787827173.626952:0:27439:0:(vvp_io.c:1889:vvp_io_init()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1787827173.626960:0:27439:0:(file.c:2170:ll_file_io_generic()) Range lock [421926, 421926] 00020000:00200000:3.0:1787827173.626965:0:27439:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 412 chunk: [1728208896, 1728209920) 1728209920, 1024 00020000:00200000:3.0:1787827173.626971:0:27439:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1787827173.626978:0:27439:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [421926, 421926] 00000008:00010000:3.0:1787827173.626986:0:27439:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1787827173.626990:0:27439:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff9e35c7d81d30, osc lock ffff9e35c2ddcc30, flags 0x0 00010000:00010000:3.0:1787827173.627001:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 20/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.627011:0:27439:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (864182272 864186367) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 20/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.627031:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 24/0,2 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.627061:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 24/0,2 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.627070:0:27439:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 23/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.627089:0:27439:0:(vvp_io.c:1339:vvp_io_write_start()) oosfile: write [1728208896, 1728209920) append 0 append lockpos 0 00000080:00200000:3.0:1787827173.627098:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421926 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.627118:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728208896, len 1024, copied 1024 00000080:00200000:3.0:1787827173.627125:0:27439:0:(rw26.c:933:ll_write_end()) page@00000000b4130021[3 000000005ce0a915 2 1 00000000d6bbf043 0 0 421926] 00000080:00200000:3.0:1787827173.627132:0:27439:0:(rw26.c:933:ll_write_end()) vmpage @00000000aa3ed607 fffffc0001001 3:0 ffff9e34c3cb41d0 421926 lru 00000080:00200000:3.0:1787827173.627142:0:27439:0:(rw26.c:933:ll_write_end()) osc-page@00000000a6b2a23c 210982: 1< 0 - - > 2< 864182272 0 0 0x0 0x0 | 00000000640821b9 0000000006873caa > 3< 0 0 > 4< 0 0 8 484458496 - | - - + - > 5< - - + - | 0 - | 55 - -> 00000080:00200000:3.0:1787827173.627146:0:27439:0:(rw26.c:933:ll_write_end()) end page@00000000b4130021 00000080:00200000:3.0:1787827173.627147:0:27439:0:(rw26.c:933:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1787827173.627150:0:27439:0:(vvp_io.c:1214:vvp_io_write_commit()) commit async pages: 1, from 0, to 1024 prio 0 00000008:00010000:3.0:1787827173.627163:0:27439:0:(osc_cache.c:565:osc_extent_merge()) ### extent: 00000000e070090a ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.627185:0:27439:0:(osc_cache.c:497:__osc_extent_remove()) ### extent: 00000000e070090a ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.627204:0:27439:0:(osc_cache.c:582:osc_extent_merge()) ### extent: 000000001063fa34 ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.627217:0:27439:0:(osc_cache.c:846:osc_extent_find()) ### extent: 000000001063fa34 ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.627235:0:27439:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff9e35d0e392f8, count 1, dirtied 1 00000080:00200000:3.0:1787827173.627240:0:27439:0:(vvp_io.c:1237:vvp_io_write_commit()) Committed 1 pages 1024 bytes, tot: 1024 00000080:00200000:3.0:1787827173.627244:0:27439:0:(vvp_io.c:1421:vvp_io_write_start()) oosfile: write bytes 0, result: 1024 00000020:00010000:3.0:1787827173.627251:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_release():245 00000020:00010000:3.0:1787827173.627255:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1787827173.627259:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c7d81d30 (ffff9e35c614ce18/1) at cl_lock_cancel():142 00010000:00010000:3.0:1787827173.627269:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.627284:0:27439:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 21/0,0 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1787827173.627302:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_fini():72 00000020:00010000:3.0:1787827173.627307:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c7d81d30 (ffff9e35e04b4e18/1) at cl_lock_fini():72 00000080:00200000:3.0:1787827173.627315:0:27439:0:(vvp_io.c:533:vvp_io_advance()) advancing 1024 bytes 00000080:00200000:3.0:1787827173.627320:0:27439:0:(file.c:2207:ll_file_io_generic()) Range unlock [421926, 421926] 00000080:00200000:3.0:1787827173.627328:0:27439:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787827173.627333:0:27439:0:(file.c:2240:ll_file_io_generic()) oosfile: 2 io complete with rc: 0, result: 1024, restart: 0 00000080:00200000:3.0:1787827173.627340:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728209920 i_blocks 3373480 00000080:00200000:3.0:1787827173.627347:0:27439:0:(file.c:2326:ll_file_io_generic()) iot: 2, result: 1024 00000080:00200008:3.0:1787827173.627358:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728209920, count: 1024 00000080:00200000:3.0:1787827173.627368:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421926 of 1024 to 1024 bytes 00000080:00200000:3.0:1787827173.627375:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728209920, len 1024, copied 1024 00000080:00200000:3.0:1787827173.627385:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728210944 i_blocks 3373480 00000080:00200000:3.0:1787827173.627392:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.627402:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728210944, count: 1024 00000080:00200000:3.0:1787827173.627410:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421926 of 2048 to 1024 bytes 00000080:00200000:3.0:1787827173.627415:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728210944, len 1024, copied 1024 00000080:00200000:3.0:1787827173.627423:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728211968 i_blocks 3373480 00000080:00200000:3.0:1787827173.627427:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.627432:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728211968, count: 1024 00000080:00200000:3.0:1787827173.627436:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421926 of 3072 to 1024 bytes 00000080:00200000:3.0:1787827173.627440:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728211968, len 1024, copied 1024 00000080:00200000:3.0:1787827173.627444:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728212992 i_blocks 3373480 00000080:00200000:3.0:1787827173.627449:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.627454:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728212992, count: 1024 00000080:00200000:3.0:1787827173.627458:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421927 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.627461:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 0, original count 1024 00000080:00200000:3.0:1787827173.627465:0:27439:0:(file.c:2090:ll_file_io_generic()) oosfile: write ppos: 1728212992, bytes: 1024 00000080:00200000:3.0:1787827173.627469:0:27439:0:(file.c:1848:ll_io_set_mirror()) oosfile: desiginated mirror: 0 00000020:00200000:3.0:1787827173.627478:0:27439:0:(cl_io.c:218:cl_io_rw_init()) header@000000007788600a[0x110, 369480, [0x200000402:0x27:0x0] hash] 00000020:00200000:3.0:1787827173.627480:0:27439:0:(cl_io.c:218:cl_io_rw_init()) io range: 2 [1728212992, 1728214016) 0 0 00000080:00200000:3.0:1787827173.627483:0:27439:0:(vvp_io.c:1889:vvp_io_init()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1787827173.627495:0:27439:0:(file.c:2170:ll_file_io_generic()) Range lock [421927, 421927] 00020000:00200000:3.0:1787827173.627500:0:27439:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 412 chunk: [1728212992, 1728214016) 1728214016, 1024 00020000:00200000:3.0:1787827173.627505:0:27439:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1787827173.627512:0:27439:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [421927, 421927] 00000008:00010000:3.0:1787827173.627519:0:27439:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1787827173.627522:0:27439:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff9e35c7d81d30, osc lock ffff9e35c2ddcc30, flags 0x0 00010000:00010000:3.0:1787827173.627532:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 20/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.627546:0:27439:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (864186368 864190463) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 20/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.627559:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 24/0,2 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.627572:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 24/0,2 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.627577:0:27439:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 23/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.627584:0:27439:0:(vvp_io.c:1339:vvp_io_write_start()) oosfile: write [1728212992, 1728214016) append 0 append lockpos 0 00000080:00200000:3.0:1787827173.627593:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421927 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.627618:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728212992, len 1024, copied 1024 00000080:00200000:3.0:1787827173.627628:0:27439:0:(rw26.c:933:ll_write_end()) page@000000004347ce23[3 000000005ce0a915 2 1 00000000d6bbf043 0 0 421927] 00000080:00200000:3.0:1787827173.627636:0:27439:0:(rw26.c:933:ll_write_end()) vmpage @000000007815f223 fffffc0001001 3:0 ffff9e34c3cb4658 421927 lru 00000080:00200000:3.0:1787827173.627652:0:27439:0:(rw26.c:933:ll_write_end()) osc-page@00000000de245fad 210983: 1< 0 - - > 2< 864186368 0 0 0x0 0x0 | 00000000640821b9 0000000006873caa > 3< 0 0 > 4< 0 0 8 484454400 - | - - + - > 5< - - + - | 0 - | 56 - -> 00000080:00200000:3.0:1787827173.627657:0:27439:0:(rw26.c:933:ll_write_end()) end page@000000004347ce23 00000080:00200000:3.0:1787827173.627659:0:27439:0:(rw26.c:933:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1787827173.627664:0:27439:0:(vvp_io.c:1214:vvp_io_write_commit()) commit async pages: 1, from 0, to 1024 prio 0 00000008:00010000:3.0:1787827173.627678:0:27439:0:(osc_cache.c:565:osc_extent_merge()) ### extent: 00000000e070090a ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.627693:0:27439:0:(osc_cache.c:497:__osc_extent_remove()) ### extent: 00000000e070090a ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.627707:0:27439:0:(osc_cache.c:582:osc_extent_merge()) ### extent: 000000001063fa34 ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.627718:0:27439:0:(osc_cache.c:846:osc_extent_find()) ### extent: 000000001063fa34 ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.627741:0:27439:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff9e35d0e392f8, count 1, dirtied 1 00000080:00200000:3.0:1787827173.627747:0:27439:0:(vvp_io.c:1237:vvp_io_write_commit()) Committed 1 pages 1024 bytes, tot: 1024 00000080:00200000:3.0:1787827173.627753:0:27439:0:(vvp_io.c:1421:vvp_io_write_start()) oosfile: write bytes 0, result: 1024 00000020:00010000:3.0:1787827173.627764:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_release():245 00000020:00010000:3.0:1787827173.627772:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1787827173.627777:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c7d81d30 (ffff9e35c614ce18/1) at cl_lock_cancel():142 00010000:00010000:3.0:1787827173.627784:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.627829:0:27439:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 21/0,0 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1787827173.627844:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_fini():72 00000020:00010000:3.0:1787827173.627849:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c7d81d30 (ffff9e35e04b4e18/1) at cl_lock_fini():72 00000080:00200000:3.0:1787827173.627857:0:27439:0:(vvp_io.c:533:vvp_io_advance()) advancing 1024 bytes 00000080:00200000:3.0:1787827173.627863:0:27439:0:(file.c:2207:ll_file_io_generic()) Range unlock [421927, 421927] 00000080:00200000:3.0:1787827173.627870:0:27439:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787827173.627875:0:27439:0:(file.c:2240:ll_file_io_generic()) oosfile: 2 io complete with rc: 0, result: 1024, restart: 0 00000080:00200000:3.0:1787827173.627884:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728214016 i_blocks 3373480 00000080:00200000:3.0:1787827173.627889:0:27439:0:(file.c:2326:ll_file_io_generic()) iot: 2, result: 1024 00000080:00200008:3.0:1787827173.627901:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728214016, count: 1024 00000080:00200000:3.0:1787827173.627910:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421927 of 1024 to 1024 bytes 00000080:00200000:3.0:1787827173.627928:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728214016, len 1024, copied 1024 00000080:00200000:3.0:1787827173.627938:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728215040 i_blocks 3373480 00000080:00200000:3.0:1787827173.627946:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.627955:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728215040, count: 1024 00000080:00200000:3.0:1787827173.627963:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421927 of 2048 to 1024 bytes 00000080:00200000:3.0:1787827173.627970:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728215040, len 1024, copied 1024 00000080:00200000:3.0:1787827173.627978:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728216064 i_blocks 3373480 00000080:00200000:3.0:1787827173.627984:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.627990:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728216064, count: 1024 00000080:00200000:3.0:1787827173.627996:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421927 of 3072 to 1024 bytes 00000080:00200000:3.0:1787827173.628001:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728216064, len 1024, copied 1024 00000080:00200000:3.0:1787827173.628008:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728217088 i_blocks 3373480 00000080:00200000:3.0:1787827173.628014:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.628019:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728217088, count: 1024 00000080:00200000:3.0:1787827173.628025:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421928 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.628029:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 0, original count 1024 00000080:00200000:3.0:1787827173.628032:0:27439:0:(file.c:2090:ll_file_io_generic()) oosfile: write ppos: 1728217088, bytes: 1024 00000080:00200000:3.0:1787827173.628035:0:27439:0:(file.c:1848:ll_io_set_mirror()) oosfile: desiginated mirror: 0 00000020:00200000:3.0:1787827173.628046:0:27439:0:(cl_io.c:218:cl_io_rw_init()) header@000000007788600a[0x110, 369481, [0x200000402:0x27:0x0] hash] 00000020:00200000:3.0:1787827173.628049:0:27439:0:(cl_io.c:218:cl_io_rw_init()) io range: 2 [1728217088, 1728218112) 0 0 00000080:00200000:3.0:1787827173.628052:0:27439:0:(vvp_io.c:1889:vvp_io_init()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1787827173.628063:0:27439:0:(file.c:2170:ll_file_io_generic()) Range lock [421928, 421928] 00020000:00200000:3.0:1787827173.628069:0:27439:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 412 chunk: [1728217088, 1728218112) 1728218112, 1024 00020000:00200000:3.0:1787827173.628075:0:27439:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1787827173.628081:0:27439:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [421928, 421928] 00000008:00010000:3.0:1787827173.628089:0:27439:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1787827173.628095:0:27439:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff9e35c7d81d30, osc lock ffff9e35c2ddcc30, flags 0x0 00010000:00010000:3.0:1787827173.628107:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 20/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.628121:0:27439:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (864190464 864194559) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 20/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.628137:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 24/0,2 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.628151:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 24/0,2 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.628170:0:27439:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 23/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.628192:0:27439:0:(vvp_io.c:1339:vvp_io_write_start()) oosfile: write [1728217088, 1728218112) append 0 append lockpos 0 00000080:00200000:3.0:1787827173.628200:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421928 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.628221:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728217088, len 1024, copied 1024 00000080:00200000:3.0:1787827173.628232:0:27439:0:(rw26.c:933:ll_write_end()) page@00000000bcb74358[3 000000005ce0a915 2 1 00000000d6bbf043 0 0 421928] 00000080:00200000:3.0:1787827173.628239:0:27439:0:(rw26.c:933:ll_write_end()) vmpage @00000000f66f779c fffffc0001001 3:0 ffff9e34c3cb43a0 421928 lru 00000080:00200000:3.0:1787827173.628253:0:27439:0:(rw26.c:933:ll_write_end()) osc-page@000000004a0a21e9 210984: 1< 0 - - > 2< 864190464 0 0 0x0 0x0 | 00000000640821b9 0000000006873caa > 3< 0 0 > 4< 0 0 8 484450304 - | - - + - > 5< - - + - | 0 - | 57 - -> 00000080:00200000:3.0:1787827173.628259:0:27439:0:(rw26.c:933:ll_write_end()) end page@00000000bcb74358 00000080:00200000:3.0:1787827173.628262:0:27439:0:(rw26.c:933:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1787827173.628265:0:27439:0:(vvp_io.c:1214:vvp_io_write_commit()) commit async pages: 1, from 0, to 1024 prio 0 00000008:00010000:3.0:1787827173.628276:0:27439:0:(osc_cache.c:565:osc_extent_merge()) ### extent: 00000000e070090a ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.628291:0:27439:0:(osc_cache.c:497:__osc_extent_remove()) ### extent: 00000000e070090a ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.628308:0:27439:0:(osc_cache.c:582:osc_extent_merge()) ### extent: 000000001063fa34 ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.628321:0:27439:0:(osc_cache.c:846:osc_extent_find()) ### extent: 000000001063fa34 ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.628342:0:27439:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff9e35d0e392f8, count 1, dirtied 1 00000080:00200000:3.0:1787827173.628350:0:27439:0:(vvp_io.c:1237:vvp_io_write_commit()) Committed 1 pages 1024 bytes, tot: 1024 00000080:00200000:3.0:1787827173.628354:0:27439:0:(vvp_io.c:1421:vvp_io_write_start()) oosfile: write bytes 0, result: 1024 00000020:00010000:3.0:1787827173.628364:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_release():245 00000020:00010000:3.0:1787827173.628371:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1787827173.628377:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c7d81d30 (ffff9e35c614ce18/1) at cl_lock_cancel():142 00010000:00010000:3.0:1787827173.628382:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.628396:0:27439:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 21/0,0 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1787827173.628410:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_fini():72 00000020:00010000:3.0:1787827173.628414:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c7d81d30 (ffff9e35e04b4e18/1) at cl_lock_fini():72 00000080:00200000:3.0:1787827173.628422:0:27439:0:(vvp_io.c:533:vvp_io_advance()) advancing 1024 bytes 00000080:00200000:3.0:1787827173.628428:0:27439:0:(file.c:2207:ll_file_io_generic()) Range unlock [421928, 421928] 00000080:00200000:3.0:1787827173.628434:0:27439:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787827173.628441:0:27439:0:(file.c:2240:ll_file_io_generic()) oosfile: 2 io complete with rc: 0, result: 1024, restart: 0 00000080:00200000:3.0:1787827173.628448:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728218112 i_blocks 3373480 00000080:00200000:3.0:1787827173.628453:0:27439:0:(file.c:2326:ll_file_io_generic()) iot: 2, result: 1024 00000080:00200008:3.0:1787827173.628464:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728218112, count: 1024 00000080:00200000:3.0:1787827173.628473:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421928 of 1024 to 1024 bytes 00000080:00200000:3.0:1787827173.628479:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728218112, len 1024, copied 1024 00000080:00200000:3.0:1787827173.628488:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728219136 i_blocks 3373480 00000080:00200000:3.0:1787827173.628493:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.628503:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728219136, count: 1024 00000080:00200000:3.0:1787827173.628512:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421928 of 2048 to 1024 bytes 00000080:00200000:3.0:1787827173.628518:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728219136, len 1024, copied 1024 00000080:00200000:3.0:1787827173.628526:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728220160 i_blocks 3373480 00000080:00200000:3.0:1787827173.628530:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.628536:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728220160, count: 1024 00000080:00200000:3.0:1787827173.628541:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421928 of 3072 to 1024 bytes 00000080:00200000:3.0:1787827173.628546:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728220160, len 1024, copied 1024 00000080:00200000:3.0:1787827173.628553:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728221184 i_blocks 3373480 00000080:00200000:3.0:1787827173.628558:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.628564:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728221184, count: 1024 00000080:00200000:3.0:1787827173.628571:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421929 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.628575:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 0, original count 1024 00000080:00200000:3.0:1787827173.628579:0:27439:0:(file.c:2090:ll_file_io_generic()) oosfile: write ppos: 1728221184, bytes: 1024 00000080:00200000:3.0:1787827173.628582:0:27439:0:(file.c:1848:ll_io_set_mirror()) oosfile: desiginated mirror: 0 00000020:00200000:3.0:1787827173.628591:0:27439:0:(cl_io.c:218:cl_io_rw_init()) header@000000007788600a[0x110, 369482, [0x200000402:0x27:0x0] hash] 00000020:00200000:3.0:1787827173.628594:0:27439:0:(cl_io.c:218:cl_io_rw_init()) io range: 2 [1728221184, 1728222208) 0 0 00000080:00200000:3.0:1787827173.628601:0:27439:0:(vvp_io.c:1889:vvp_io_init()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1787827173.628613:0:27439:0:(file.c:2170:ll_file_io_generic()) Range lock [421929, 421929] 00020000:00200000:3.0:1787827173.628618:0:27439:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 412 chunk: [1728221184, 1728222208) 1728222208, 1024 00020000:00200000:3.0:1787827173.628625:0:27439:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1787827173.628633:0:27439:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [421929, 421929] 00000008:00010000:3.0:1787827173.628642:0:27439:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1787827173.628647:0:27439:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff9e35c7d81d30, osc lock ffff9e35c2ddcc30, flags 0x0 00010000:00010000:3.0:1787827173.628660:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 20/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.628674:0:27439:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (864194560 864198655) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 20/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.628684:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 24/0,2 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.628705:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 24/0,2 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.628728:0:27439:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 23/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.628745:0:27439:0:(vvp_io.c:1339:vvp_io_write_start()) oosfile: write [1728221184, 1728222208) append 0 append lockpos 0 00000080:00200000:3.0:1787827173.628752:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421929 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.628775:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728221184, len 1024, copied 1024 00000080:00200000:3.0:1787827173.628786:0:27439:0:(rw26.c:933:ll_write_end()) page@0000000054ffea6e[3 000000005ce0a915 2 1 00000000d6bbf043 0 0 421929] 00000080:00200000:3.0:1787827173.628794:0:27439:0:(rw26.c:933:ll_write_end()) vmpage @00000000391d9b34 fffffc0001001 3:0 ffff9e34c3cb4d98 421929 lru 00000080:00200000:3.0:1787827173.628829:0:27439:0:(rw26.c:933:ll_write_end()) osc-page@00000000a9c952a9 210985: 1< 0 - - > 2< 864194560 0 0 0x0 0x0 | 00000000640821b9 0000000006873caa > 3< 0 0 > 4< 0 0 8 484446208 - | - - + - > 5< - - + - | 0 - | 58 - -> 00000080:00200000:3.0:1787827173.628834:0:27439:0:(rw26.c:933:ll_write_end()) end page@0000000054ffea6e 00000080:00200000:3.0:1787827173.628838:0:27439:0:(rw26.c:933:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1787827173.628842:0:27439:0:(vvp_io.c:1214:vvp_io_write_commit()) commit async pages: 1, from 0, to 1024 prio 0 00000008:00010000:3.0:1787827173.628853:0:27439:0:(osc_cache.c:565:osc_extent_merge()) ### extent: 00000000e070090a ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.628877:0:27439:0:(osc_cache.c:497:__osc_extent_remove()) ### extent: 00000000e070090a ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.628900:0:27439:0:(osc_cache.c:582:osc_extent_merge()) ### extent: 000000001063fa34 ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.628933:0:27439:0:(osc_cache.c:846:osc_extent_find()) ### extent: 000000001063fa34 ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.628957:0:27439:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff9e35d0e392f8, count 1, dirtied 1 00000080:00200000:3.0:1787827173.628964:0:27439:0:(vvp_io.c:1237:vvp_io_write_commit()) Committed 1 pages 1024 bytes, tot: 1024 00000080:00200000:3.0:1787827173.628968:0:27439:0:(vvp_io.c:1421:vvp_io_write_start()) oosfile: write bytes 0, result: 1024 00000020:00010000:3.0:1787827173.628978:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_release():245 00000020:00010000:3.0:1787827173.628984:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1787827173.628989:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c7d81d30 (ffff9e35c614ce18/1) at cl_lock_cancel():142 00010000:00010000:3.0:1787827173.628998:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.629016:0:27439:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 21/0,0 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1787827173.629034:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_fini():72 00000020:00010000:3.0:1787827173.629041:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c7d81d30 (ffff9e35e04b4e18/1) at cl_lock_fini():72 00000080:00200000:3.0:1787827173.629049:0:27439:0:(vvp_io.c:533:vvp_io_advance()) advancing 1024 bytes 00000080:00200000:3.0:1787827173.629054:0:27439:0:(file.c:2207:ll_file_io_generic()) Range unlock [421929, 421929] 00000080:00200000:3.0:1787827173.629061:0:27439:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787827173.629069:0:27439:0:(file.c:2240:ll_file_io_generic()) oosfile: 2 io complete with rc: 0, result: 1024, restart: 0 00000080:00200000:3.0:1787827173.629079:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728222208 i_blocks 3373480 00000080:00200000:3.0:1787827173.629086:0:27439:0:(file.c:2326:ll_file_io_generic()) iot: 2, result: 1024 00000080:00200008:3.0:1787827173.629097:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728222208, count: 1024 00000080:00200000:3.0:1787827173.629106:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421929 of 1024 to 1024 bytes 00000080:00200000:3.0:1787827173.629112:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728222208, len 1024, copied 1024 00000080:00200000:3.0:1787827173.629121:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728223232 i_blocks 3373480 00000080:00200000:3.0:1787827173.629128:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.629134:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728223232, count: 1024 00000080:00200000:3.0:1787827173.629143:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421929 of 2048 to 1024 bytes 00000080:00200000:3.0:1787827173.629149:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728223232, len 1024, copied 1024 00000080:00200000:3.0:1787827173.629156:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728224256 i_blocks 3373480 00000080:00200000:3.0:1787827173.629163:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.629170:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728224256, count: 1024 00000080:00200000:3.0:1787827173.629176:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421929 of 3072 to 1024 bytes 00000080:00200000:3.0:1787827173.629182:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728224256, len 1024, copied 1024 00000080:00200000:3.0:1787827173.629188:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728225280 i_blocks 3373480 00000080:00200000:3.0:1787827173.629195:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.629203:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728225280, count: 1024 00000080:00200000:3.0:1787827173.629210:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421930 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.629215:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 0, original count 1024 00000080:00200000:3.0:1787827173.629218:0:27439:0:(file.c:2090:ll_file_io_generic()) oosfile: write ppos: 1728225280, bytes: 1024 00000080:00200000:3.0:1787827173.629222:0:27439:0:(file.c:1848:ll_io_set_mirror()) oosfile: desiginated mirror: 0 00000020:00200000:3.0:1787827173.629226:0:27439:0:(cl_io.c:218:cl_io_rw_init()) header@000000007788600a[0x110, 369483, [0x200000402:0x27:0x0] hash] 00000020:00200000:3.0:1787827173.629228:0:27439:0:(cl_io.c:218:cl_io_rw_init()) io range: 2 [1728225280, 1728226304) 0 0 00000080:00200000:3.0:1787827173.629231:0:27439:0:(vvp_io.c:1889:vvp_io_init()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1787827173.629238:0:27439:0:(file.c:2170:ll_file_io_generic()) Range lock [421930, 421930] 00020000:00200000:3.0:1787827173.629244:0:27439:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 412 chunk: [1728225280, 1728226304) 1728226304, 1024 00020000:00200000:3.0:1787827173.629250:0:27439:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1787827173.629257:0:27439:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [421930, 421930] 00000008:00010000:3.0:1787827173.629266:0:27439:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1787827173.629271:0:27439:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff9e35c7d81d30, osc lock ffff9e35c2ddcc30, flags 0x0 00010000:00010000:3.0:1787827173.629282:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 20/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.629302:0:27439:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (864198656 864202751) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 20/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.629324:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 24/0,2 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.629353:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 24/0,2 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.629366:0:27439:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 23/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.629380:0:27439:0:(vvp_io.c:1339:vvp_io_write_start()) oosfile: write [1728225280, 1728226304) append 0 append lockpos 0 00000080:00200000:3.0:1787827173.629390:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421930 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.629421:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728225280, len 1024, copied 1024 00000080:00200000:3.0:1787827173.629430:0:27439:0:(rw26.c:933:ll_write_end()) page@00000000cc112bc0[3 000000005ce0a915 2 1 00000000d6bbf043 0 0 421930] 00000080:00200000:3.0:1787827173.629437:0:27439:0:(rw26.c:933:ll_write_end()) vmpage @00000000a92abd51 fffffc0001001 3:0 ffff9e34c3cb52b8 421930 lru 00000080:00200000:3.0:1787827173.629448:0:27439:0:(rw26.c:933:ll_write_end()) osc-page@000000006b5b2bf2 210986: 1< 0 - - > 2< 864198656 0 0 0x0 0x0 | 00000000640821b9 0000000006873caa > 3< 0 0 > 4< 0 0 8 484442112 - | - - + - > 5< - - + - | 0 - | 59 - -> 00000080:00200000:3.0:1787827173.629452:0:27439:0:(rw26.c:933:ll_write_end()) end page@00000000cc112bc0 00000080:00200000:3.0:1787827173.629455:0:27439:0:(rw26.c:933:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1787827173.629460:0:27439:0:(vvp_io.c:1214:vvp_io_write_commit()) commit async pages: 1, from 0, to 1024 prio 0 00000008:00010000:3.0:1787827173.629474:0:27439:0:(osc_cache.c:565:osc_extent_merge()) ### extent: 00000000e070090a ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.629490:0:27439:0:(osc_cache.c:497:__osc_extent_remove()) ### extent: 00000000e070090a ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.629507:0:27439:0:(osc_cache.c:582:osc_extent_merge()) ### extent: 000000001063fa34 ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.629526:0:27439:0:(osc_cache.c:846:osc_extent_find()) ### extent: 000000001063fa34 ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.629549:0:27439:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff9e35d0e392f8, count 1, dirtied 1 00000080:00200000:3.0:1787827173.629556:0:27439:0:(vvp_io.c:1237:vvp_io_write_commit()) Committed 1 pages 1024 bytes, tot: 1024 00000080:00200000:3.0:1787827173.629561:0:27439:0:(vvp_io.c:1421:vvp_io_write_start()) oosfile: write bytes 0, result: 1024 00000020:00010000:3.0:1787827173.629571:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_release():245 00000020:00010000:3.0:1787827173.629578:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1787827173.629582:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c7d81d30 (ffff9e35c614ce18/1) at cl_lock_cancel():142 00010000:00010000:3.0:1787827173.629590:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.629605:0:27439:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 21/0,0 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1787827173.629627:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_fini():72 00000020:00010000:3.0:1787827173.629633:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c7d81d30 (ffff9e35e04b4e18/1) at cl_lock_fini():72 00000080:00200000:3.0:1787827173.629642:0:27439:0:(vvp_io.c:533:vvp_io_advance()) advancing 1024 bytes 00000080:00200000:3.0:1787827173.629647:0:27439:0:(file.c:2207:ll_file_io_generic()) Range unlock [421930, 421930] 00000080:00200000:3.0:1787827173.629654:0:27439:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787827173.629659:0:27439:0:(file.c:2240:ll_file_io_generic()) oosfile: 2 io complete with rc: 0, result: 1024, restart: 0 00000080:00200000:3.0:1787827173.629669:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728226304 i_blocks 3373480 00000080:00200000:3.0:1787827173.629676:0:27439:0:(file.c:2326:ll_file_io_generic()) iot: 2, result: 1024 00000080:00200008:3.0:1787827173.629688:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728226304, count: 1024 00000080:00200000:3.0:1787827173.629698:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421930 of 1024 to 1024 bytes 00000080:00200000:3.0:1787827173.629703:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728226304, len 1024, copied 1024 00000080:00200000:3.0:1787827173.629709:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728227328 i_blocks 3373480 00000080:00200000:3.0:1787827173.629712:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.629718:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728227328, count: 1024 00000080:00200000:3.0:1787827173.629722:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421930 of 2048 to 1024 bytes 00000080:00200000:3.0:1787827173.629727:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728227328, len 1024, copied 1024 00000080:00200000:3.0:1787827173.629735:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728228352 i_blocks 3373480 00000080:00200000:3.0:1787827173.629742:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.629749:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728228352, count: 1024 00000080:00200000:3.0:1787827173.629756:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421930 of 3072 to 1024 bytes 00000080:00200000:3.0:1787827173.629762:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728228352, len 1024, copied 1024 00000080:00200000:3.0:1787827173.629768:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728229376 i_blocks 3373480 00000080:00200000:3.0:1787827173.629771:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.629777:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728229376, count: 1024 00000080:00200000:3.0:1787827173.629784:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421931 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.629788:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 0, original count 1024 00000080:00200000:3.0:1787827173.629792:0:27439:0:(file.c:2090:ll_file_io_generic()) oosfile: write ppos: 1728229376, bytes: 1024 00000080:00200000:3.0:1787827173.629818:0:27439:0:(file.c:1848:ll_io_set_mirror()) oosfile: desiginated mirror: 0 00000020:00200000:3.0:1787827173.629828:0:27439:0:(cl_io.c:218:cl_io_rw_init()) header@000000007788600a[0x110, 369484, [0x200000402:0x27:0x0] hash] 00000020:00200000:3.0:1787827173.629831:0:27439:0:(cl_io.c:218:cl_io_rw_init()) io range: 2 [1728229376, 1728230400) 0 0 00000080:00200000:3.0:1787827173.629837:0:27439:0:(vvp_io.c:1889:vvp_io_init()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1787827173.629846:0:27439:0:(file.c:2170:ll_file_io_generic()) Range lock [421931, 421931] 00020000:00200000:3.0:1787827173.629851:0:27439:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 412 chunk: [1728229376, 1728230400) 1728230400, 1024 00020000:00200000:3.0:1787827173.629854:0:27439:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1787827173.629864:0:27439:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [421931, 421931] 00000008:00010000:3.0:1787827173.629873:0:27439:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1787827173.629878:0:27439:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff9e35c7d81d30, osc lock ffff9e35c2ddcc30, flags 0x0 00010000:00010000:3.0:1787827173.629892:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 20/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.629926:0:27439:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (864202752 864206847) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 20/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.629949:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 24/0,2 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.629976:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 24/0,2 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.629991:0:27439:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 23/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.630006:0:27439:0:(vvp_io.c:1339:vvp_io_write_start()) oosfile: write [1728229376, 1728230400) append 0 append lockpos 0 00000080:00200000:3.0:1787827173.630013:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421931 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.630035:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728229376, len 1024, copied 1024 00000080:00200000:3.0:1787827173.630044:0:27439:0:(rw26.c:933:ll_write_end()) page@000000001a77e777[3 000000005ce0a915 2 1 00000000d6bbf043 0 0 421931] 00000080:00200000:3.0:1787827173.630052:0:27439:0:(rw26.c:933:ll_write_end()) vmpage @0000000090d73a81 fffffc0001001 3:0 ffff9e34c3cb5488 421931 lru 00000080:00200000:3.0:1787827173.630062:0:27439:0:(rw26.c:933:ll_write_end()) osc-page@000000007c9ee36a 210987: 1< 0 - - > 2< 864202752 0 0 0x0 0x0 | 00000000640821b9 0000000006873caa > 3< 0 0 > 4< 0 0 8 484438016 - | - - + - > 5< - - + - | 0 - | 60 - -> 00000080:00200000:3.0:1787827173.630067:0:27439:0:(rw26.c:933:ll_write_end()) end page@000000001a77e777 00000080:00200000:3.0:1787827173.630071:0:27439:0:(rw26.c:933:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1787827173.630075:0:27439:0:(vvp_io.c:1214:vvp_io_write_commit()) commit async pages: 1, from 0, to 1024 prio 0 00000008:00010000:3.0:1787827173.630087:0:27439:0:(osc_cache.c:565:osc_extent_merge()) ### extent: 00000000e070090a ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.630104:0:27439:0:(osc_cache.c:497:__osc_extent_remove()) ### extent: 00000000e070090a ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.630122:0:27439:0:(osc_cache.c:582:osc_extent_merge()) ### extent: 000000001063fa34 ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.630139:0:27439:0:(osc_cache.c:846:osc_extent_find()) ### extent: 000000001063fa34 ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.630158:0:27439:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff9e35d0e392f8, count 1, dirtied 1 00000080:00200000:3.0:1787827173.630162:0:27439:0:(vvp_io.c:1237:vvp_io_write_commit()) Committed 1 pages 1024 bytes, tot: 1024 00000080:00200000:3.0:1787827173.630165:0:27439:0:(vvp_io.c:1421:vvp_io_write_start()) oosfile: write bytes 0, result: 1024 00000020:00010000:3.0:1787827173.630174:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_release():245 00000020:00010000:3.0:1787827173.630178:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1787827173.630182:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c7d81d30 (ffff9e35c614ce18/1) at cl_lock_cancel():142 00010000:00010000:3.0:1787827173.630189:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.630201:0:27439:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 21/0,0 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1787827173.630225:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_fini():72 00000020:00010000:3.0:1787827173.630230:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c7d81d30 (ffff9e35e04b4e18/1) at cl_lock_fini():72 00000080:00200000:3.0:1787827173.630239:0:27439:0:(vvp_io.c:533:vvp_io_advance()) advancing 1024 bytes 00000080:00200000:3.0:1787827173.630245:0:27439:0:(file.c:2207:ll_file_io_generic()) Range unlock [421931, 421931] 00000080:00200000:3.0:1787827173.630252:0:27439:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787827173.630257:0:27439:0:(file.c:2240:ll_file_io_generic()) oosfile: 2 io complete with rc: 0, result: 1024, restart: 0 00000080:00200000:3.0:1787827173.630266:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728230400 i_blocks 3373480 00000080:00200000:3.0:1787827173.630271:0:27439:0:(file.c:2326:ll_file_io_generic()) iot: 2, result: 1024 00000080:00200008:3.0:1787827173.630281:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728230400, count: 1024 00000080:00200000:3.0:1787827173.630289:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421931 of 1024 to 1024 bytes 00000080:00200000:3.0:1787827173.630292:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728230400, len 1024, copied 1024 00000080:00200000:3.0:1787827173.630299:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728231424 i_blocks 3373480 00000080:00200000:3.0:1787827173.630302:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.630309:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728231424, count: 1024 00000080:00200000:3.0:1787827173.630318:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421931 of 2048 to 1024 bytes 00000080:00200000:3.0:1787827173.630326:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728231424, len 1024, copied 1024 00000080:00200000:3.0:1787827173.630333:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728232448 i_blocks 3373480 00000080:00200000:3.0:1787827173.630338:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.630345:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728232448, count: 1024 00000080:00200000:3.0:1787827173.630350:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421931 of 3072 to 1024 bytes 00000080:00200000:3.0:1787827173.630356:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728232448, len 1024, copied 1024 00000080:00200000:3.0:1787827173.630365:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728233472 i_blocks 3373480 00000080:00200000:3.0:1787827173.630371:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.630378:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728233472, count: 1024 00000080:00200000:3.0:1787827173.630385:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421932 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.630389:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 0, original count 1024 00000080:00200000:3.0:1787827173.630393:0:27439:0:(file.c:2090:ll_file_io_generic()) oosfile: write ppos: 1728233472, bytes: 1024 00000080:00200000:3.0:1787827173.630397:0:27439:0:(file.c:1848:ll_io_set_mirror()) oosfile: desiginated mirror: 0 00000020:00200000:3.0:1787827173.630407:0:27439:0:(cl_io.c:218:cl_io_rw_init()) header@000000007788600a[0x110, 369485, [0x200000402:0x27:0x0] hash] 00000020:00200000:3.0:1787827173.630411:0:27439:0:(cl_io.c:218:cl_io_rw_init()) io range: 2 [1728233472, 1728234496) 0 0 00000080:00200000:3.0:1787827173.630414:0:27439:0:(vvp_io.c:1889:vvp_io_init()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1787827173.630420:0:27439:0:(file.c:2170:ll_file_io_generic()) Range lock [421932, 421932] 00020000:00200000:3.0:1787827173.630425:0:27439:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 412 chunk: [1728233472, 1728234496) 1728234496, 1024 00020000:00200000:3.0:1787827173.630430:0:27439:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1787827173.630437:0:27439:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [421932, 421932] 00000008:00010000:3.0:1787827173.630446:0:27439:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1787827173.630451:0:27439:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff9e35c7d81d30, osc lock ffff9e35c2ddcc30, flags 0x0 00010000:00010000:3.0:1787827173.630465:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 20/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.630492:0:27439:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (864206848 864210943) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 20/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.630510:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 24/0,2 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.630537:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 24/0,2 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.630550:0:27439:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 23/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.630566:0:27439:0:(vvp_io.c:1339:vvp_io_write_start()) oosfile: write [1728233472, 1728234496) append 0 append lockpos 0 00000080:00200000:3.0:1787827173.630574:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421932 of 0 to 1024 bytes 00000080:00200000:3.0:1787827173.630602:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728233472, len 1024, copied 1024 00000080:00200000:3.0:1787827173.630611:0:27439:0:(rw26.c:933:ll_write_end()) page@000000006db2daf5[3 000000005ce0a915 2 1 00000000d6bbf043 0 0 421932] 00000080:00200000:3.0:1787827173.630617:0:27439:0:(rw26.c:933:ll_write_end()) vmpage @00000000b587163b fffffc0001021 2:0 ffff9e34c3cb5bc8 421932 lru 00000080:00200000:3.0:1787827173.630628:0:27439:0:(rw26.c:933:ll_write_end()) osc-page@000000001be255af 210988: 1< 0 - - > 2< 864206848 0 0 0x0 0x0 | 00000000640821b9 0000000006873caa > 3< 0 0 > 4< 0 0 8 484433920 - | - - + - > 5< - - + - | 0 - | 61 - -> 00000080:00200000:3.0:1787827173.630632:0:27439:0:(rw26.c:933:ll_write_end()) end page@000000006db2daf5 00000080:00200000:3.0:1787827173.630634:0:27439:0:(rw26.c:933:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1787827173.630639:0:27439:0:(vvp_io.c:1214:vvp_io_write_commit()) commit async pages: 1, from 0, to 1024 prio 0 00000008:00010000:3.0:1787827173.630652:0:27439:0:(osc_cache.c:565:osc_extent_merge()) ### extent: 00000000e070090a ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.630668:0:27439:0:(osc_cache.c:497:__osc_extent_remove()) ### extent: 00000000e070090a ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.630684:0:27439:0:(osc_cache.c:582:osc_extent_merge()) ### extent: 000000001063fa34 ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827173.630696:0:27439:0:(osc_cache.c:846:osc_extent_find()) ### extent: 000000001063fa34 ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827173.630716:0:27439:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff9e35d0e392f8, count 1, dirtied 1 00000080:00200000:3.0:1787827173.630721:0:27439:0:(vvp_io.c:1237:vvp_io_write_commit()) Committed 1 pages 1024 bytes, tot: 1024 00000080:00200000:3.0:1787827173.630726:0:27439:0:(vvp_io.c:1421:vvp_io_write_start()) oosfile: write bytes 0, result: 1024 00000020:00010000:3.0:1787827173.630737:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_release():245 00000020:00010000:3.0:1787827173.630744:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1787827173.630751:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9e35c7d81d30 (ffff9e35c614ce18/1) at cl_lock_cancel():142 00010000:00010000:3.0:1787827173.630759:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827173.630782:0:27439:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 21/0,0 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1787827173.630816:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35d1202c68 (ffff9e35e04b4e18/0) at cl_lock_fini():72 00000020:00010000:3.0:1787827173.630821:0:27439:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9e35c7d81d30 (ffff9e35e04b4e18/1) at cl_lock_fini():72 00000080:00200000:3.0:1787827173.630826:0:27439:0:(vvp_io.c:533:vvp_io_advance()) advancing 1024 bytes 00000080:00200000:3.0:1787827173.630831:0:27439:0:(file.c:2207:ll_file_io_generic()) Range unlock [421932, 421932] 00000080:00200000:3.0:1787827173.630837:0:27439:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787827173.630845:0:27439:0:(file.c:2240:ll_file_io_generic()) oosfile: 2 io complete with rc: 0, result: 1024, restart: 0 00000080:00200000:3.0:1787827173.630854:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728234496 i_blocks 3373480 00000080:00200000:3.0:1787827173.630858:0:27439:0:(file.c:2326:ll_file_io_generic()) iot: 2, result: 1024 00000080:00200008:3.0:1787827173.630867:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728234496, count: 1024 00000080:00200000:3.0:1787827173.630875:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421932 of 1024 to 1024 bytes 00000080:00200000:3.0:1787827173.630879:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728234496, len 1024, copied 1024 00000080:00200000:3.0:1787827173.630885:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1728235520 i_blocks 3373480 00000080:00200000:3.0:1787827173.630889:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 1024, original count 1024 00000080:00200008:3.0:1787827173.630898:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1728235520, count: 1024 00000080:00200000:3.0:1787827173.630906:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 421932 of 2048 to 1024 bytes 00000080:00200000:3.0:1787827173.630928:0:27439:0:(rw26.c:875:ll_write_end()) pos 1728235520, len 1024, copied 1024 00000080:00200000:3.0:1787827173.754185:0:71:0:(vvp_io.c:1889:vvp_io_init()) [0x200000403:0x11:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:3.0:1787827173.755107:0:71:0:(vvp_io.c:304:vvp_io_fini()) [0x200000403:0x11:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787827173.755128:0:71:0:(vvp_io.c:1889:vvp_io_init()) [0x200000402:0x27:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:3.0:1787827173.755912:0:71:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x27:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787827173.758064:0:71:0:(vvp_io.c:1889:vvp_io_init()) [0x200000403:0x11:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:3.0:1787827173.758964:0:71:0:(vvp_io.c:304:vvp_io_fini()) [0x200000403:0x11:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787827173.758989:0:71:0:(vvp_io.c:1889:vvp_io_init()) [0x200000402:0x27:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:3.0:1787827173.759622:0:71:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x27:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787827173.759631:0:71:0:(vvp_io.c:1889:vvp_io_init()) [0x200000403:0x11:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:3.0:1787827173.760334:0:71:0:(vvp_io.c:304:vvp_io_fini()) [0x200000403:0x11:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787827173.760353:0:71:0:(vvp_io.c:1889:vvp_io_init()) [0x200000402:0x27:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:3.0:1787827173.761169:0:71:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x27:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787827173.761193:0:71:0:(vvp_io.c:1889:vvp_io_init()) [0x200000403:0x11:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:3.0:1787827173.761961:0:71:0:(vvp_io.c:304:vvp_io_fini()) [0x200000403:0x11:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787827173.763836:0:33:0:(vvp_io.c:1889:vvp_io_init()) [0x200000402:0x27:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:3.0:1787827173.810195:0:33:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x27:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787827173.810232:0:33:0:(vvp_io.c:1889:vvp_io_init()) [0x200000403:0x11:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:3.0:1787827173.861747:0:33:0:(vvp_io.c:304:vvp_io_fini()) [0x200000403:0x11:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787827173.861788:0:33:0:(vvp_io.c:1889:vvp_io_init()) [0x200000402:0x27:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:3.0:1787827173.910359:0:33:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x27:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787827173.910399:0:33:0:(vvp_io.c:1889:vvp_io_init()) [0x200000403:0x11:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:3.0:1787827173.961601:0:33:0:(vvp_io.c:304:vvp_io_fini()) [0x200000403:0x11:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787827173.961645:0:33:0:(vvp_io.c:1889:vvp_io_init()) [0x200000403:0x11:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:3.0:1787827174.008891:0:33:0:(vvp_io.c:304:vvp_io_fini()) [0x200000403:0x11:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787827174.165695:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1732244480 i_blocks 3373480 00000080:00200000:3.0:1787827174.165708:0:27439:0:(file.c:2326:ll_file_io_generic()) iot: 2, result: 1024 00000080:00200008:3.0:1787827174.165740:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1732244480, count: 1024 00000080:00200000:3.0:1787827174.165751:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 422911 of 1024 to 1024 bytes 00000080:00200000:3.0:1787827174.165762:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 0, original count 1024 00000080:00200000:3.0:1787827174.165767:0:27439:0:(file.c:2090:ll_file_io_generic()) oosfile: write ppos: 1732244480, bytes: 1024 00000080:00200000:3.0:1787827174.165772:0:27439:0:(file.c:1848:ll_io_set_mirror()) oosfile: desiginated mirror: 0 00000020:00200000:3.0:1787827174.165783:0:27439:0:(cl_io.c:218:cl_io_rw_init()) header@000000007788600a[0x110, 353577, [0x200000402:0x27:0x0] hash] 00000020:00200000:3.0:1787827174.165786:0:27439:0:(cl_io.c:218:cl_io_rw_init()) io range: 2 [1732244480, 1732245504) 0 0 00000080:00200000:3.0:1787827174.165793:0:27439:0:(vvp_io.c:1889:vvp_io_init()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1787827174.165857:0:27439:0:(file.c:2170:ll_file_io_generic()) Range lock [422911, 422911] 00020000:00200000:3.0:1787827174.165866:0:27439:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 412 chunk: [1732244480, 1732245504) 1732245504, 1024 00020000:00200000:3.0:1787827174.165878:0:27439:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1787827174.165897:0:27439:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [422911, 422911] 00000008:00010000:3.0:1787827174.165928:0:27439:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1787827174.165936:0:27439:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff9e35c7d81d30, osc lock ffff9e35c2ddcc30, flags 0x0 00010000:00010000:3.0:1787827174.166038:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 20/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827174.166061:0:27439:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (868216832 868220927) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 20/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827174.166092:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 24/0,2 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827174.166136:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 24/0,2 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827174.166182:0:27439:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 23/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827174.166233:0:27439:0:(vvp_io.c:1339:vvp_io_write_start()) oosfile: write [1732244480, 1732245504) append 0 append lockpos 0 00000080:00200000:3.0:1787827174.166247:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 422911 of 1024 to 1024 bytes 00000080:00200000:3.0:1787827174.188007:0:27439:0:(rw26.c:875:ll_write_end()) pos 1732244480, len 1024, copied 1024 00000080:00200000:3.0:1787827174.188031:0:27439:0:(rw26.c:933:ll_write_end()) page@00000000815e6239[3 000000005ce0a915 2 1 00000000d2544009 0 0 422911] 00000080:00200000:3.0:1787827174.188039:0:27439:0:(rw26.c:933:ll_write_end()) vmpage @000000007d771302 17ffffc0001009 4:0 ffff9e35c5fb1828 422911 lru 00000080:00200000:3.0:1787827174.188057:0:27439:0:(rw26.c:933:ll_write_end()) osc-page@000000007b5589ea 211967: 1< 2 - - > 2< 868216832 0 1024 0x0 0x40420 | 00000000640821b9 0000000006873caa > 3< 0 0 > 4< 0 0 8 484638720 - | - - + - > 5< - - + - | 0 - | 17 - -> 00000080:00200000:3.0:1787827174.188064:0:27439:0:(rw26.c:933:ll_write_end()) end page@00000000815e6239 00000080:00200000:3.0:1787827174.188069:0:27439:0:(rw26.c:933:ll_write_end()) queued cl_page: 1. 00000100:00100000:3.0:1787827174.732317:0:2412:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c49ea580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:66d74899-006e-4207-beb4-e14ddbbfbbe2:2412:1874672085321728:192.168.203.141@tcp:4:dd.0 00000008:00010000:3.0:1787827174.751106:0:2412:0:(osc_cache.c:908:osc_extent_finish()) ### extent: 0000000068875f36 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1787827174.752133:0:2412:0:(osc_cache.c:497:__osc_extent_remove()) ### extent: 0000000068875f36 ns: lustre-OST0000-osc-ffff9e35d0749800 lock: 0000000064d5bbce/0x55f45c2fc9174972 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0xb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ed5 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000100:00100000:3.0:1787827174.752184:0:2412:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c49ea580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:66d74899-006e-4207-beb4-e14ddbbfbbe2:2412:1874672085321728:192.168.203.141@tcp:4:dd.0 00000080:00200000:3.0:1787827175.079914:0:27439:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x27:0x0] updating i_size 1740633088 i_blocks 3389864 00000080:00200000:3.0:1787827175.079919:0:27439:0:(file.c:2326:ll_file_io_generic()) iot: 2, result: 1024 00000080:00200008:3.0:1787827175.079927:0:27439:0:(file.c:2652:do_file_write_iter()) START file oosfile:[0x200000402:0x27:0x0], ppos: 1740633088, count: 1024 00000080:00200000:3.0:1787827175.079932:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 424959 of 1024 to 1024 bytes 00000080:00200000:3.0:1787827175.079935:0:27439:0:(file.c:2628:ll_do_tiny_write()) result: 0, original count 1024 00000080:00200000:3.0:1787827175.079936:0:27439:0:(file.c:2090:ll_file_io_generic()) oosfile: write ppos: 1740633088, bytes: 1024 00000080:00200000:3.0:1787827175.079939:0:27439:0:(file.c:1848:ll_io_set_mirror()) oosfile: desiginated mirror: 0 00000020:00200000:3.0:1787827175.079942:0:27439:0:(cl_io.c:218:cl_io_rw_init()) header@000000007788600a[0x110, 355625, [0x200000402:0x27:0x0] hash] 00000020:00200000:3.0:1787827175.079944:0:27439:0:(cl_io.c:218:cl_io_rw_init()) io range: 2 [1740633088, 1740634112) 0 0 00000080:00200000:3.0:1787827175.079946:0:27439:0:(vvp_io.c:1889:vvp_io_init()) [0x200000402:0x27:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1787827175.079950:0:27439:0:(file.c:2170:ll_file_io_generic()) Range lock [424959, 424959] 00020000:00200000:3.0:1787827175.079953:0:27439:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 414 chunk: [1740633088, 1740634112) 1740634112, 1024 00020000:00200000:3.0:1787827175.079955:0:27439:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1787827175.079960:0:27439:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [424959, 424959] 00000008:00010000:3.0:1787827175.079966:0:27439:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1787827175.079968:0:27439:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff9e35c7d81cb0, osc lock ffff9e35c2ddc000, flags 0x0 00010000:00010000:3.0:1787827175.079979:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 22/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827175.080000:0:27439:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (872411136 872415231) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 21/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827175.080016:0:27439:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 25/0,2 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827175.080021:0:27439:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 25/0,2 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1787827175.080026:0:27439:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff9e35d0749800 lock: 00000000be575d53/0x55f45c2fc917495d lrc: 24/0,1 mode: PW/PW res: [0x2c0000401:0xa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xfc2f462b47f41ec0 expref: -99 pid: 27439 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1787827175.080030:0:27439:0:(vvp_io.c:1339:vvp_io_write_start()) oosfile: write [1740633088, 1740634112) append 0 append lockpos 0 00000080:00200000:3.0:1787827175.080034:0:27439:0:(rw26.c:681:ll_write_begin()) Writing 424959 of 1024 to 1024 bytes 00010000:00010000:3.0:1787827176.071998:0:15664:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827178.119994:0:6167:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827181.191969:0:15664:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827183.239903:0:6167:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827186.311934:0:15664:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827188.359886:0:6167:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827189.256003:0:2412:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c49ebc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2412:1874672085324928:192.168.203.141@tcp:400:kworker.0 00000100:00100000:3.0:1787827189.256110:0:2412:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c49e92c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:66d74899-006e-4207-beb4-e14ddbbfbbe2:2412:1874672085325184:192.168.203.141@tcp:400:kworker.0 00000100:00100000:3.0:1787827189.256167:0:2412:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c49e9a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:66d74899-006e-4207-beb4-e14ddbbfbbe2:2412:1874672085325440:192.168.203.141@tcp:400:kworker.0 00000100:00100000:3.0:1787827189.256213:0:2412:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c49eb840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:5a777d8d-8e74-43dd-b307-248963eb234b:2412:1874672085325696:192.168.203.141@tcp:400:kworker.0 00000100:00100000:3.0:1787827189.256259:0:2412:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c49ea940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:5a777d8d-8e74-43dd-b307-248963eb234b:2412:1874672085325952:192.168.203.141@tcp:400:kworker.0 00000100:00100000:3.0:1787827190.294411:0:2411:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c49e8f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2411:1874672085326080:192.168.203.141@tcp:400:kworker.0 00000100:00100000:3.0:1787827190.294555:0:2411:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:66d74899-006e-4207-beb4-e14ddbbfbbe2:2411:1874672085326336:192.168.203.141@tcp:400:kworker.0 00000100:00100000:3.0:1787827190.294627:0:2411:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:66d74899-006e-4207-beb4-e14ddbbfbbe2:2411:1874672085326592:192.168.203.141@tcp:400:kworker.0 00000100:00100000:3.0:1787827190.294703:0:2411:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:5a777d8d-8e74-43dd-b307-248963eb234b:2411:1874672085326848:192.168.203.141@tcp:400:kworker.0 00010000:00010000:3.0:1787827191.431991:0:15664:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827192.342406:0:2411:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2411:1874672085328256:192.168.203.141@tcp:400:kworker.0 00000100:00100000:3.0:1787827192.342523:0:2411:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:66d74899-006e-4207-beb4-e14ddbbfbbe2:2411:1874672085328512:192.168.203.141@tcp:400:kworker.0 00000100:00100000:3.0:1787827192.342567:0:2411:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:5a777d8d-8e74-43dd-b307-248963eb234b:2411:1874672085328768:192.168.203.141@tcp:400:kworker.0 00010000:00010000:3.0:1787827193.479964:0:6167:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827195.415153:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c64983c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085329920:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787827195.415224:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085330048:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787827195.415267:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c64992c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085330176:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787827195.423837:0:2411:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2411:1874672085328256:192.168.203.141@tcp:400:kworker.0 00000100:00100000:3.0:1787827195.423862:0:2411:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c49e8f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2411:1874672085326080:192.168.203.141@tcp:400:kworker.0 00010000:00010000:3.0:1787827196.551975:0:15664:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827198.599952:0:6167:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827201.671992:0:15664:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827203.719978:0:6167:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000400:3.0:1787827205.639965:0:2412:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1787827189/real 1787827189] req@ffff9e35c49e9a40 x1874672085325440/t0(0) o400->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 224/224 e 0 to 1 dl 1787827205 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827205.640034:0:2412:0:(client.c:2547:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff9e35c49e9a40 x1874672085325440/t0(0) o400->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 224/224 e 0 to 1 dl 1787827205 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827205.640067:0:2412:0:(import.c:214:ptlrpc_set_import_discon()) osc: import ffff9e35d02f9000 already not connected (conn 4, was 2): CONNECTING 00000100:00000400:3.0:1787827205.640081:0:2412:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1787827189/real 1787827189] req@ffff9e35c49eb840 x1874672085325696/t0(0) o400->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 224/224 e 0 to 1 dl 1787827205 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827205.640117:0:2412:0:(client.c:2547:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff9e35c49eb840 x1874672085325696/t0(0) o400->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 224/224 e 0 to 1 dl 1787827205 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827205.640133:0:2412:0:(import.c:214:ptlrpc_set_import_discon()) mdc: import ffff9e35c8e03800 already not connected (conn 2, was 1): CONNECTING 00000100:00000400:3.0:1787827205.640148:0:2412:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1787827189/real 1787827189] req@ffff9e35c49ea940 x1874672085325952/t0(0) o400->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 224/224 e 0 to 1 dl 1787827205 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827205.640172:0:2412:0:(client.c:2547:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff9e35c49ea940 x1874672085325952/t0(0) o400->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 224/224 e 0 to 1 dl 1787827205 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827205.640184:0:2412:0:(import.c:214:ptlrpc_set_import_discon()) osc: import ffff9e35c8e00000 already not connected (conn 3, was 1): CONNECTING 00000100:00000400:3.0:1787827205.640197:0:2412:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1787827189/real 1787827189] req@ffff9e35c49e92c0 x1874672085325184/t0(0) o400->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 224/224 e 0 to 1 dl 1787827205 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827205.640246:0:2412:0:(client.c:2547:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff9e35c49e92c0 x1874672085325184/t0(0) o400->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 224/224 e 0 to 1 dl 1787827205 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827205.640262:0:2412:0:(import.c:214:ptlrpc_set_import_discon()) mdc: import ffff9e35c60f7800 already not connected (conn 2, was 1): CONNECTING 00000100:00100000:3.0:1787827205.640283:0:2412:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c49e9a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:66d74899-006e-4207-beb4-e14ddbbfbbe2:2412:1874672085325440:192.168.203.141@tcp:400:kworker.0 00000100:00100000:3.0:1787827205.640296:0:2412:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c49eb840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:5a777d8d-8e74-43dd-b307-248963eb234b:2412:1874672085325696:192.168.203.141@tcp:400:kworker.0 00000100:00100000:3.0:1787827205.640309:0:2412:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c49ea940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:5a777d8d-8e74-43dd-b307-248963eb234b:2412:1874672085325952:192.168.203.141@tcp:400:kworker.0 00000100:00100000:3.0:1787827205.640320:0:2412:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c49e92c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:66d74899-006e-4207-beb4-e14ddbbfbbe2:2412:1874672085325184:192.168.203.141@tcp:400:kworker.0 00000100:00000400:3.0:1787827206.663923:0:2412:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1787827190/real 1787827190] req@ffff9e35c49ea1c0 x1874672085326208/t0(0) o400->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 224/224 e 0 to 1 dl 1787827206 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827206.663982:0:2412:0:(client.c:2547:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff9e35c49ea1c0 x1874672085326208/t0(0) o400->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 224/224 e 0 to 1 dl 1787827206 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827206.664009:0:2412:0:(import.c:214:ptlrpc_set_import_discon()) mdc: import ffff9e35c8ec9800 already not connected (conn 3, was 1): CONNECTING 00000100:00000400:3.0:1787827206.664024:0:2412:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1787827190/real 1787827190] req@ffff9e35c6498f00 x1874672085326464/t0(0) o400->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 224/224 e 0 to 1 dl 1787827206 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827206.664048:0:2412:0:(client.c:2547:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff9e35c6498f00 x1874672085326464/t0(0) o400->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 224/224 e 0 to 1 dl 1787827206 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827206.664059:0:2412:0:(import.c:214:ptlrpc_set_import_discon()) osc: import ffff9e35c60f1000 already not connected (conn 4, was 2): CONNECTING 00000100:00000400:3.0:1787827206.664072:0:2412:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1787827190/real 1787827190] req@ffff9e35c649a580 x1874672085326720/t0(0) o400->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 224/224 e 0 to 1 dl 1787827206 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827206.664090:0:2412:0:(client.c:2547:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff9e35c649a580 x1874672085326720/t0(0) o400->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 224/224 e 0 to 1 dl 1787827206 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827206.664096:0:2412:0:(import.c:214:ptlrpc_set_import_discon()) mdc: import ffff9e35c8e00800 already not connected (conn 3, was 1): CONNECTING 00000100:00000400:3.0:1787827206.664104:0:2412:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1787827190/real 1787827190] req@ffff9e35c6499680 x1874672085327104/t0(0) o400->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 224/224 e 0 to 1 dl 1787827206 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827206.664118:0:2412:0:(client.c:2547:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff9e35c6499680 x1874672085327104/t0(0) o400->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 224/224 e 0 to 1 dl 1787827206 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827206.664128:0:2412:0:(import.c:214:ptlrpc_set_import_discon()) osc: import ffff9e35c8e00000 already not connected (conn 3, was 1): CONNECTING 00000100:00100000:3.0:1787827206.664141:0:2412:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c49ea1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:66d74899-006e-4207-beb4-e14ddbbfbbe2:2412:1874672085326208:192.168.203.141@tcp:400:kworker.0 00000100:00100000:3.0:1787827206.664146:0:2412:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:66d74899-006e-4207-beb4-e14ddbbfbbe2:2412:1874672085326464:192.168.203.141@tcp:400:kworker.0 00000100:00100000:3.0:1787827206.664150:0:2412:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:5a777d8d-8e74-43dd-b307-248963eb234b:2412:1874672085326720:192.168.203.141@tcp:400:kworker.0 00000100:00100000:3.0:1787827206.664154:0:2412:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:5a777d8d-8e74-43dd-b307-248963eb234b:2412:1874672085327104:192.168.203.141@tcp:400:kworker.0 00010000:00010000:3.0:1787827206.791970:0:15664:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000400:3.0:1787827208.711949:0:2412:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1787827192/real 1787827192] req@ffff9e35c6498780 x1874672085328384/t0(0) o400->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 224/224 e 0 to 1 dl 1787827208 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827208.712024:0:2412:0:(client.c:2547:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff9e35c6498780 x1874672085328384/t0(0) o400->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 224/224 e 0 to 1 dl 1787827208 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827208.712035:0:2412:0:(import.c:214:ptlrpc_set_import_discon()) mdc: import ffff9e35c8ec9800 already not connected (conn 3, was 1): CONNECTING 00000100:00000400:3.0:1787827208.712043:0:2412:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1787827192/real 1787827192] req@ffff9e35c6498b40 x1874672085328640/t0(0) o400->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 224/224 e 0 to 1 dl 1787827208 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827208.712060:0:2412:0:(client.c:2547:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff9e35c6498b40 x1874672085328640/t0(0) o400->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 224/224 e 0 to 1 dl 1787827208 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827208.712069:0:2412:0:(import.c:214:ptlrpc_set_import_discon()) osc: import ffff9e35c60f1000 already not connected (conn 4, was 2): CONNECTING 00000100:00000400:3.0:1787827208.712075:0:2412:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1787827192/real 1787827192] req@ffff9e35c649a940 x1874672085328896/t0(0) o400->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 224/224 e 0 to 1 dl 1787827208 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827208.712084:0:2412:0:(client.c:2547:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff9e35c649a940 x1874672085328896/t0(0) o400->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 224/224 e 0 to 1 dl 1787827208 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827208.712090:0:2412:0:(import.c:214:ptlrpc_set_import_discon()) mdc: import ffff9e35c8e03800 already not connected (conn 3, was 1): CONNECTING 00000100:00100000:3.0:1787827208.712100:0:2412:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:66d74899-006e-4207-beb4-e14ddbbfbbe2:2412:1874672085328384:192.168.203.141@tcp:400:kworker.0 00000100:00100000:3.0:1787827208.712107:0:2412:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:66d74899-006e-4207-beb4-e14ddbbfbbe2:2412:1874672085328640:192.168.203.141@tcp:400:kworker.0 00000100:00100000:3.0:1787827208.712113:0:2412:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:5a777d8d-8e74-43dd-b307-248963eb234b:2412:1874672085328896:192.168.203.141@tcp:400:kworker.0 00010000:00010000:3.0:1787827208.839992:0:6167:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827211.911925:0:15664:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827213.959930:0:6167:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827217.031984:0:15664:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827219.079974:0:6167:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827222.151925:0:15664:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827224.199992:0:6167:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827227.271929:0:15664:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827229.319932:0:6167:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000100:3.0:1787827231.752170:0:2400:0:(socklnd_cb.c:2254:ksocknal_find_timed_out_conn()) A connection with 12345-192.168.203.141@tcp (192.168.203.141:988) timed out; the network or node may be down. 00010000:00010000:3.0:1787827232.391983:0:15664:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827234.439961:0:6167:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827237.511935:0:15664:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827239.559945:0:6167:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827242.631989:0:15664:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1787827243.592056:0:2398:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1021 -> 192.168.203.141:988 00000400:00000100:3.0:1787827243.592101:0:2398:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.141@tcp at host 192.168.203.141 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1787827243.592117:0:2398:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1787827243.592173:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000800:00000100:3.0:1787827243.592186:0:2398:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1787827243.592193:0:2398:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1787827243.592201:0:2398:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1787827243.592208:0:2398:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1787827243.592215:0:2398:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1787827243.592222:0:2398:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1787827243.592229:0:2398:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00010000:00010000:3.0:1787827244.679945:0:6167:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1787827246.664041:0:2399:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1021 -> 192.168.203.141:988 00000400:00000100:3.0:1787827246.664076:0:2399:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.141@tcp at host 192.168.203.141 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1787827246.664088:0:2399:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1787827246.664121:0:2399:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000800:00000100:3.0:1787827246.664130:0:2399:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1787827246.664137:0:2399:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1787827246.664145:0:2399:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1787827246.664151:0:2399:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1787827246.664159:0:2399:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1787827246.664165:0:2399:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1787827246.664183:0:2399:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00010000:00010000:3.0:1787827249.799930:0:6167:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827254.919936:0:6167:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827255.816061:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085337088:192.168.203.141@tcp:250:kworker.0 00000800:00000100:3.0:1787827255.816133:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827255.816186:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827255/real 1787827255] req@ffff9e35c498c3c0 x1874672085337088/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827280 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827255.816227:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c3c0 x1874672085337088/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827280 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827255.816245:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827255.816254:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827255.816264:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085337088:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1787827255.816296:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085337216:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787827255.816331:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827255.816367:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827255/real 1787827255] req@ffff9e35c498d2c0 x1874672085337216/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827280 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827255.816404:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498d2c0 x1874672085337216/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827280 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827255.816424:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827255.816432:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827255.816446:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085337216:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787827255.816459:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085337344:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787827255.816493:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827255.816517:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827255/real 1787827255] req@ffff9e35c498e1c0 x1874672085337344/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827280 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827255.816552:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e1c0 x1874672085337344/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827280 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827255.816573:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827255.816582:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827255.816589:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085337344:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787827255.816602:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085337472:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787827255.816634:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827255.816729:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827255/real 1787827255] req@ffff9e35c498cf00 x1874672085337472/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827280 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827255.816742:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498cf00 x1874672085337472/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827280 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827255.816751:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827255.816755:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827255.816758:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085337472:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787827255.816778:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085337600:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787827255.816828:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827255.816842:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827255/real 1787827255] req@ffff9e35c498da40 x1874672085337600/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827280 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827255.816854:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498da40 x1874672085337600/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827280 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827255.816868:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827255.816872:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827255.816874:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085337600:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787827255.816880:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085337728:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787827255.816902:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827255.816914:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827255/real 1787827255] req@ffff9e35c498ed00 x1874672085337728/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827280 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827255.816926:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498ed00 x1874672085337728/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827280 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827255.816934:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827255.816937:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827255.816940:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085337728:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787827255.816945:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085337856:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787827255.816962:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827255.816972:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827255/real 1787827255] req@ffff9e35c498de00 x1874672085337856/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827280 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827255.816984:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498de00 x1874672085337856/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827280 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827255.816992:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827255.816995:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827255.816999:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085337856:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787827255.817006:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085337984:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787827255.817024:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827255.817032:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827255/real 1787827255] req@ffff9e35c498e940 x1874672085337984/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827280 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827255.817041:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e940 x1874672085337984/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827280 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827255.817050:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827255.817053:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827255.817056:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085337984:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787827255.817063:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085338112:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787827255.817081:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827255.817092:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827255/real 1787827255] req@ffff9e35c498f840 x1874672085338112/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827280 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827255.817104:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f840 x1874672085338112/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827280 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827255.817112:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827255.817116:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827255.817120:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085338112:192.168.203.141@tcp:8:kworker.0 00010000:00010000:3.0:1787827260.039964:0:6167:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827260.936139:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085338240:192.168.203.141@tcp:250:kworker.0 00000800:00000100:3.0:1787827260.936215:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827260.936268:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827260/real 1787827260] req@ffff9e35da6f5a40 x1874672085338240/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827285 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827260.936320:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5a40 x1874672085338240/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827285 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827260.936334:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827260.936339:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827260.936343:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085338240:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1787827260.936356:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085338368:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787827260.936394:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827260.936417:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827260/real 1787827260] req@ffff9e35da6f61c0 x1874672085338368/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827285 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827260.936442:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f61c0 x1874672085338368/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827285 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827260.936453:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827260.936462:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827260.936472:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085338368:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787827260.936511:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085338496:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787827260.936544:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827260.936564:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827260/real 1787827260] req@ffff9e35da6f43c0 x1874672085338496/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827285 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827260.936602:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f43c0 x1874672085338496/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827285 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827260.936613:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827260.936617:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827260.936621:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085338496:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787827260.936630:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085338624:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787827260.936652:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827260.936664:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827260/real 1787827260] req@ffff9e35da6f5680 x1874672085338624/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827285 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827260.936677:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5680 x1874672085338624/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827285 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827260.936686:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827260.936689:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827260.936693:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085338624:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787827260.936699:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085338752:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787827260.936719:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827260.936843:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827260/real 1787827260] req@ffff9e35da6f70c0 x1874672085338752/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827285 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827260.936858:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f70c0 x1874672085338752/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827285 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827260.936871:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827260.936874:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827260.936878:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085338752:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787827260.936884:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085338880:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787827260.936906:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827260.936917:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827260/real 1787827260] req@ffff9e35da6f6940 x1874672085338880/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827285 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827260.936929:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6940 x1874672085338880/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827285 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827260.936945:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827260.936948:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827260.936951:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085338880:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787827260.936957:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085339008:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787827260.936978:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827260.936990:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827260/real 1787827260] req@ffff9e35da6f5e00 x1874672085339008/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827285 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827260.937003:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5e00 x1874672085339008/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827285 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827260.937011:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827260.937014:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827260.937018:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085339008:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787827260.937040:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085339136:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787827260.937062:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827260.937074:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827260/real 1787827260] req@ffff9e35da6f7840 x1874672085339136/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827285 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827260.937088:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7840 x1874672085339136/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827285 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827260.937097:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827260.937101:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827260.937105:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085339136:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787827260.937112:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085339264:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787827260.937139:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827260.937151:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827260/real 1787827260] req@ffff9e35da6f4b40 x1874672085339264/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827285 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827260.937164:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4b40 x1874672085339264/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827285 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827260.937173:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827260.937177:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827260.937181:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085339264:192.168.203.141@tcp:8:kworker.0 00010000:00010000:3.0:1787827265.160021:0:6167:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827266.056127:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085339392:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1787827266.056248:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085339520:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787827266.056303:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085339648:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787827266.056333:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f52c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085339776:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787827266.056362:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085339904:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787827266.056401:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085340032:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787827266.056445:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085340160:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787827266.056474:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085340288:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787827266.056495:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085340416:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787827269.128297:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827266/real 1787827269] req@ffff9e35da6f7480 x1874672085339392/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827291 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827269.128338:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7480 x1874672085339392/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827291 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827269.128358:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827269.128369:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827269.128378:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085339392:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1787827269.128404:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827266/real 1787827269] req@ffff9e35da6f4000 x1874672085339520/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827291 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827269.128434:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4000 x1874672085339520/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827291 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827269.128454:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827269.128462:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827269.128467:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085339520:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787827269.128491:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827266/real 1787827269] req@ffff9e35da6f6d00 x1874672085340160/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827291 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827269.128505:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6d00 x1874672085340160/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827291 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827269.128513:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827269.128518:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827269.128521:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085340160:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787827269.128527:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827266/real 1787827269] req@ffff9e35da6f4f00 x1874672085340288/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827291 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827269.128539:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4f00 x1874672085340288/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827291 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827269.128547:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827269.128550:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827269.128553:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085340288:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787827269.128560:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827266/real 1787827269] req@ffff9e35da6f4b40 x1874672085340416/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827291 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827269.128573:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4b40 x1874672085340416/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827291 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827269.128581:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827269.128584:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827269.128587:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085340416:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787827269.128594:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827266/real 1787827269] req@ffff9e35da6f4780 x1874672085339648/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827291 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827269.128605:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4780 x1874672085339648/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827291 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827269.128613:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827269.128616:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827269.128620:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085339648:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787827269.128626:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827266/real 1787827269] req@ffff9e35da6f52c0 x1874672085339776/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827291 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827269.128637:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f52c0 x1874672085339776/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827291 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827269.128645:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827269.128649:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827269.128653:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f52c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085339776:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787827269.128660:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827266/real 1787827269] req@ffff9e35da6f6580 x1874672085339904/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827291 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827269.128672:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6580 x1874672085339904/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827291 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827269.128681:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827269.128685:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827269.128688:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085339904:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787827269.128695:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827266/real 1787827269] req@ffff9e35da6f7c00 x1874672085340032/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827291 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827269.128707:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7c00 x1874672085340032/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827291 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827269.128715:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827269.128719:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827269.128722:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085340032:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1787827270.279986:0:6167:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827271.176114:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085340544:192.168.203.141@tcp:250:kworker.0 00000800:00000100:3.0:1787827271.176222:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827271.176324:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827271/real 1787827271] req@ffff9e35c498e580 x1874672085340544/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827296 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827271.176363:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e580 x1874672085340544/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827296 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827271.176385:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827271.176404:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827271.176423:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085340544:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1787827271.176445:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085340672:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787827271.176501:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827271.176531:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827271/real 1787827271] req@ffff9e35c498c000 x1874672085340672/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827296 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827271.176578:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c000 x1874672085340672/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827296 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827271.176604:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827271.176618:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827271.176628:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085340672:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787827271.176697:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085340800:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787827271.176730:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827271.176746:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827271/real 1787827271] req@ffff9e35c498f480 x1874672085340800/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827296 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827271.176759:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f480 x1874672085340800/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827296 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827271.176770:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827271.176775:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827271.176778:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085340800:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787827271.176787:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085340928:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787827271.177166:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827271.177181:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827271/real 1787827271] req@ffff9e35c498d680 x1874672085340928/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827296 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827271.177201:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498d680 x1874672085340928/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827296 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827271.177211:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827271.177214:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827271.177217:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085340928:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787827271.177224:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085341056:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787827271.177263:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827271.177276:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827271/real 1787827271] req@ffff9e35c498f0c0 x1874672085341056/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827296 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827271.177289:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f0c0 x1874672085341056/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827296 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827271.177298:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827271.177302:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827271.177306:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085341056:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787827271.177313:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085341184:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787827271.177336:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827271.177349:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827271/real 1787827271] req@ffff9e35c498f840 x1874672085341184/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827296 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827271.177361:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f840 x1874672085341184/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827296 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827271.177376:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827271.177380:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827271.177384:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085341184:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787827271.177391:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085341312:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787827271.177419:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827271.177431:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827271/real 1787827271] req@ffff9e35c498e940 x1874672085341312/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827296 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827271.177441:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e940 x1874672085341312/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827296 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827271.177449:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827271.177452:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827271.177455:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085341312:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787827271.177461:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085341440:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787827271.177480:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827271.177490:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827271/real 1787827271] req@ffff9e35c498de00 x1874672085341440/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827296 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827271.177500:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498de00 x1874672085341440/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827296 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827271.177508:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827271.177512:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827271.177515:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085341440:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787827271.177522:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085341568:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787827271.177540:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827271.177553:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827271/real 1787827271] req@ffff9e35c498ed00 x1874672085341568/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827296 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827271.177564:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498ed00 x1874672085341568/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827296 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827271.177572:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827271.177575:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827271.177578:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085341568:192.168.203.141@tcp:8:kworker.0 00010000:00010000:3.0:1787827275.400009:0:6167:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827276.296021:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085341696:192.168.203.141@tcp:250:kworker.0 00000800:00000100:3.0:1787827276.296090:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827276.296140:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827276/real 1787827276] req@ffff9e35c498da40 x1874672085341696/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827301 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827276.296177:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498da40 x1874672085341696/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827301 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827276.296290:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827276.296396:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827276.296403:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085341696:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1787827276.296475:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085341824:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787827276.296505:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827276.296520:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827276/real 1787827276] req@ffff9e35c498cf00 x1874672085341824/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827301 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827276.296532:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498cf00 x1874672085341824/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827301 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827276.296538:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827276.296540:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827276.296542:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085341824:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787827276.296547:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085341952:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787827276.296561:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827276.296567:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827276/real 1787827276] req@ffff9e35c498e1c0 x1874672085341952/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827301 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827276.296575:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e1c0 x1874672085341952/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827301 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827276.296580:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827276.296582:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827276.296584:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085341952:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787827276.296587:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085342080:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787827276.296598:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827276.296604:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827276/real 1787827276] req@ffff9e35c498d2c0 x1874672085342080/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827301 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827276.296615:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498d2c0 x1874672085342080/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827301 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827276.296620:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827276.296622:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827276.296624:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085342080:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787827276.296628:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085342208:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787827276.296638:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827276.296643:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827276/real 1787827276] req@ffff9e35c498c3c0 x1874672085342208/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827301 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827276.296650:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c3c0 x1874672085342208/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827301 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827276.296656:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827276.296658:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827276.296660:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085342208:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787827276.296663:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085342336:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787827276.296672:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827276.296678:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827276/real 1787827276] req@ffff9e35c498c780 x1874672085342336/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827301 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827276.296685:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c780 x1874672085342336/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827301 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827276.296693:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827276.296696:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827276.296698:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085342336:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787827276.296701:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085342464:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787827276.296710:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827276.296716:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827276/real 1787827276] req@ffff9e35c498fc00 x1874672085342464/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827301 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827276.296722:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498fc00 x1874672085342464/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827301 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827276.296727:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827276.296729:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827276.296731:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085342464:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787827276.296734:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085342592:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787827276.296744:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827276.296749:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827276/real 1787827276] req@ffff9e35c498cb40 x1874672085342592/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827301 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827276.296756:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498cb40 x1874672085342592/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827301 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827276.296761:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827276.296763:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827276.296765:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085342592:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787827276.296768:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085342720:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787827276.296780:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827276.296791:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827276/real 1787827276] req@ffff9e35c498ed00 x1874672085342720/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827301 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827276.296848:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498ed00 x1874672085342720/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827301 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827276.296858:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827276.296864:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827276.296871:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085342720:192.168.203.141@tcp:8:kworker.0 00010000:00010000:3.0:1787827280.519968:0:6167:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827281.416043:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085342848:192.168.203.141@tcp:250:kworker.0 00000800:00000100:3.0:1787827281.416104:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827281.416141:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827281/real 1787827281] req@ffff9e35c498de00 x1874672085342848/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827306 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827281.416195:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498de00 x1874672085342848/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827306 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827281.416211:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827281.416218:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827281.416225:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085342848:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1787827281.416248:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085342976:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787827281.416283:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827281.416301:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827281/real 1787827281] req@ffff9e35c498e940 x1874672085342976/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827306 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827281.416316:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e940 x1874672085342976/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827306 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827281.416324:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827281.416328:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827281.416331:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085342976:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787827281.416343:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085343104:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787827281.416362:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827281.416374:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827281/real 1787827281] req@ffff9e35c498f840 x1874672085343104/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827306 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827281.416390:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f840 x1874672085343104/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827306 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827281.416398:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827281.416405:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827281.416411:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085343104:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787827281.416422:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085343232:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787827281.416445:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827281.416457:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827281/real 1787827281] req@ffff9e35c498f0c0 x1874672085343232/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827306 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827281.416479:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f0c0 x1874672085343232/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827306 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827281.416512:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827281.416523:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827281.416533:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085343232:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787827281.416561:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085343360:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787827281.416596:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827281.416605:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827281/real 1787827281] req@ffff9e35c498d680 x1874672085343360/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827306 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827281.416613:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498d680 x1874672085343360/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827306 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827281.416619:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827281.416621:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827281.416623:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085343360:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787827281.416627:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085343488:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787827281.416637:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827281.416643:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827281/real 1787827281] req@ffff9e35c498f480 x1874672085343488/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827306 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827281.416650:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f480 x1874672085343488/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827306 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827281.416659:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827281.416661:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827281.416663:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085343488:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787827281.416667:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085343616:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787827281.416682:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827281.416689:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827281/real 1787827281] req@ffff9e35c498c000 x1874672085343616/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827306 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827281.416696:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c000 x1874672085343616/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827306 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827281.416701:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827281.416703:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827281.416705:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085343616:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787827281.416709:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085343744:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787827281.416725:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827281.416735:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827281/real 1787827281] req@ffff9e35c498e580 x1874672085343744/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827306 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827281.416748:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e580 x1874672085343744/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827306 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827281.416757:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827281.416760:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827281.416764:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085343744:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787827281.416778:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085343872:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787827281.416793:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827281.416821:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827281/real 1787827281] req@ffff9e35c498e940 x1874672085343872/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827306 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827281.416829:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e940 x1874672085343872/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827306 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827281.416835:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827281.416837:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827281.416839:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085343872:192.168.203.141@tcp:8:kworker.0 00010000:00010000:3.0:1787827285.639908:0:6167:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827286.536004:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085344000:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1787827286.536103:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085344128:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787827286.536161:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085344256:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787827286.536190:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085344384:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787827286.536229:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085344512:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787827286.536257:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085344640:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787827286.536280:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085344768:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787827286.536306:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085344896:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787827286.536324:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085345024:192.168.203.141@tcp:8:kworker.0 00000400:00000100:3.0:1787827289.096367:0:2398:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1021 -> 192.168.203.141:988 00000400:00000100:3.0:1787827289.096431:0:2398:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.141@tcp at host 192.168.203.141 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1787827289.096451:0:2398:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1787827289.096536:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000800:00000100:3.0:1787827289.096566:0:2398:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1787827289.096594:0:2398:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1787827289.096615:0:2398:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1787827289.096643:0:2398:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1787827289.096673:0:2398:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1787827289.096716:0:2398:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1787827289.096746:0:2398:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00010000:00010000:3.0:1787827290.759929:0:6167:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827291.656174:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085345152:192.168.203.141@tcp:250:kworker.0 00000800:00000100:3.0:1787827291.656280:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827291.656344:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827291/real 1787827291] req@ffff9e35c498e1c0 x1874672085345152/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827316 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827291.656381:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e1c0 x1874672085345152/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827316 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827291.656437:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827291.656447:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827291.656459:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085345152:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1787827291.656508:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085345280:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787827291.656571:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827291.656613:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827291/real 1787827291] req@ffff9e35c498de00 x1874672085345280/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827316 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827291.656665:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498de00 x1874672085345280/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827316 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827291.656688:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827291.656692:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827291.656698:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085345280:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787827291.656710:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085345408:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787827291.656749:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827291.656763:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827291/real 1787827291] req@ffff9e35c498fc00 x1874672085345408/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827316 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827291.656778:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498fc00 x1874672085345408/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827316 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827291.656787:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827291.656842:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827291.656846:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085345408:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787827291.656854:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085345536:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787827291.656877:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827291.656889:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827291/real 1787827291] req@ffff9e35c498cb40 x1874672085345536/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827316 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827291.656910:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498cb40 x1874672085345536/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827316 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827291.656919:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827291.656923:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827291.656927:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085345536:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787827291.656934:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085345664:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787827291.656955:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827291.656973:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827291/real 1787827291] req@ffff9e35c498ed00 x1874672085345664/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827316 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827291.656987:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498ed00 x1874672085345664/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827316 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827291.656996:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827291.657000:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827291.657003:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085345664:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787827291.657009:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085345792:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787827291.657028:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827291.657040:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827291/real 1787827291] req@ffff9e35c498d2c0 x1874672085345792/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827316 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827291.657052:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498d2c0 x1874672085345792/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827316 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827291.657068:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827291.657071:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827291.657074:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085345792:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787827291.657081:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085345920:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787827291.657122:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827291.657136:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827291/real 1787827291] req@ffff9e35c498c3c0 x1874672085345920/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827316 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827291.657150:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c3c0 x1874672085345920/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827316 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827291.657159:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827291.657163:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827291.657167:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085345920:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787827291.657194:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085346048:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787827291.657218:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827291.657231:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827291/real 1787827291] req@ffff9e35c498c780 x1874672085346048/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827316 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827291.657245:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c780 x1874672085346048/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827316 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827291.657254:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827291.657258:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827291.657262:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085346048:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787827291.657269:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085346176:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787827291.657290:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827291.657303:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827291/real 1787827291] req@ffff9e35c498cf00 x1874672085346176/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827316 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827291.657315:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498cf00 x1874672085346176/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827316 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827291.657325:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827291.657328:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827291.657332:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085346176:192.168.203.141@tcp:8:kworker.0 00010000:00010000:3.0:1787827295.879898:0:6167:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827296.776048:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085346304:192.168.203.141@tcp:250:kworker.0 00000800:00000100:3.0:1787827296.776124:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827296.776184:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827296/real 1787827296] req@ffff9e35c498da40 x1874672085346304/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827321 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827296.776202:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498da40 x1874672085346304/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827321 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827296.776218:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827296.776221:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827296.776226:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085346304:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1787827296.776249:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085346432:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787827296.776278:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827296.776290:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827296/real 1787827296] req@ffff9e35c498cf00 x1874672085346432/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827321 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827296.776307:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498cf00 x1874672085346432/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827321 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827296.776320:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827296.776327:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827296.776331:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085346432:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787827296.776339:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085346560:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787827296.776359:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827296.776370:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827296/real 1787827296] req@ffff9e35c498c780 x1874672085346560/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827321 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827296.776390:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c780 x1874672085346560/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827321 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827296.776406:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827296.776414:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827296.776420:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085346560:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787827296.776427:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085346688:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787827296.776442:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827296.776452:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827296/real 1787827296] req@ffff9e35c498c3c0 x1874672085346688/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827321 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827296.776465:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c3c0 x1874672085346688/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827321 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827296.776470:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827296.776472:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827296.776474:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085346688:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787827296.776478:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085346816:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787827296.776487:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827296.776501:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827296/real 1787827296] req@ffff9e35c498d2c0 x1874672085346816/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827321 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827296.776512:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498d2c0 x1874672085346816/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827321 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827296.776518:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827296.776520:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827296.776522:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085346816:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787827296.776526:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085346944:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787827296.776546:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827296.776553:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827296/real 1787827296] req@ffff9e35c498ed00 x1874672085346944/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827321 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827296.776560:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498ed00 x1874672085346944/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827321 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827296.776569:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827296.776571:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827296.776573:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085346944:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787827296.776586:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085347072:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787827296.776596:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827296.776602:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827296/real 1787827296] req@ffff9e35c498cb40 x1874672085347072/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827321 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827296.776609:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498cb40 x1874672085347072/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827321 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827296.776614:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827296.776616:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827296.776618:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085347072:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787827296.776621:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085347200:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787827296.776630:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827296.776636:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827296/real 1787827296] req@ffff9e35c498fc00 x1874672085347200/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827321 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827296.776643:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498fc00 x1874672085347200/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827321 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827296.776648:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827296.776650:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827296.776652:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085347200:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787827296.776655:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085347328:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787827296.776664:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827296.776677:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827296/real 1787827296] req@ffff9e35c498de00 x1874672085347328/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827321 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827296.776684:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498de00 x1874672085347328/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827321 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827296.776688:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827296.776690:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827296.776692:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085347328:192.168.203.141@tcp:8:kworker.0 00010000:00010000:3.0:1787827300.999961:0:6167:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827301.896045:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085347456:192.168.203.141@tcp:250:kworker.0 00000800:00000100:3.0:1787827301.896106:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827301.896143:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827301/real 1787827301] req@ffff9e35c498e1c0 x1874672085347456/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827326 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827301.896166:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e1c0 x1874672085347456/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827326 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827301.896176:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827301.896181:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827301.896187:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085347456:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1787827301.896211:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085347584:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787827301.896240:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827301.896252:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827301/real 1787827301] req@ffff9e35c498e940 x1874672085347584/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827326 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827301.896274:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e940 x1874672085347584/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827326 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827301.896286:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827301.896293:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827301.896301:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085347584:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787827301.896311:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085347712:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787827301.896332:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827301.896344:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827301/real 1787827301] req@ffff9e35c498e580 x1874672085347712/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827326 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827301.896364:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e580 x1874672085347712/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827326 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827301.896384:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827301.896392:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827301.896402:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085347712:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787827301.896418:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085347840:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787827301.896460:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827301.896480:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827301/real 1787827301] req@ffff9e35c498c000 x1874672085347840/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827326 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827301.896503:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c000 x1874672085347840/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827326 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827301.896513:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827301.896517:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827301.896520:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085347840:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787827301.896527:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085347968:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787827301.896549:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827301.896561:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827301/real 1787827301] req@ffff9e35c498f480 x1874672085347968/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827326 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827301.896573:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f480 x1874672085347968/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827326 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827301.896581:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827301.896585:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827301.896589:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085347968:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787827301.896608:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085348096:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787827301.896631:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827301.896643:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827301/real 1787827301] req@ffff9e35c498d680 x1874672085348096/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827326 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827301.896666:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498d680 x1874672085348096/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827326 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827301.896681:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827301.896685:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827301.896688:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085348096:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787827301.896695:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085348224:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787827301.896716:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827301.896728:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827301/real 1787827301] req@ffff9e35c498f0c0 x1874672085348224/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827326 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827301.896739:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f0c0 x1874672085348224/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827326 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827301.896747:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827301.896751:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827301.896754:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085348224:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787827301.896761:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085348352:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787827301.896781:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827301.896793:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827301/real 1787827301] req@ffff9e35c498f840 x1874672085348352/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827326 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827301.896832:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f840 x1874672085348352/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827326 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827301.896842:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827301.896845:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827301.896849:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085348352:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787827301.896856:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085348480:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787827301.896876:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827301.896889:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827301/real 1787827301] req@ffff9e35c498e1c0 x1874672085348480/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827326 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827301.896901:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e1c0 x1874672085348480/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827326 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827301.896910:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827301.896913:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827301.896916:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085348480:192.168.203.141@tcp:8:kworker.0 00010000:00010000:3.0:1787827306.119926:0:6167:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827307.016115:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085348608:192.168.203.141@tcp:250:kworker.0 00000800:00000100:3.0:1787827307.016204:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827307.016273:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827307/real 1787827307] req@ffff9e35da6f7840 x1874672085348608/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827332 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827307.016320:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7840 x1874672085348608/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827332 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827307.016360:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827307.016371:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827307.016380:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085348608:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1787827307.016394:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085348736:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787827307.016461:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827307.016491:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827307/real 1787827307] req@ffff9e35da6f5e00 x1874672085348736/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827332 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827307.016502:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5e00 x1874672085348736/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827332 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827307.016510:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827307.016513:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827307.016515:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085348736:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787827307.016543:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085348864:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787827307.016561:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827307.016569:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827307/real 1787827307] req@ffff9e35da6f6940 x1874672085348864/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827332 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827307.016576:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6940 x1874672085348864/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827332 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827307.016582:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827307.016585:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827307.016587:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085348864:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787827307.016591:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085348992:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787827307.016602:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827307.016608:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827307/real 1787827307] req@ffff9e35da6f70c0 x1874672085348992/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827332 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827307.016620:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f70c0 x1874672085348992/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827332 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827307.016625:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827307.016627:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827307.016629:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085348992:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787827307.016633:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085349120:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787827307.016644:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827307.016650:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827307/real 1787827307] req@ffff9e35da6f5680 x1874672085349120/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827332 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827307.016657:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5680 x1874672085349120/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827332 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827307.016672:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827307.016675:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827307.016677:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085349120:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787827307.016681:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085349248:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787827307.016694:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827307.016700:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827307/real 1787827307] req@ffff9e35da6f43c0 x1874672085349248/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827332 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827307.016708:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f43c0 x1874672085349248/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827332 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827307.016718:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827307.016720:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827307.016722:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085349248:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787827307.016725:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085349376:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787827307.016735:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827307.016741:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827307/real 1787827307] req@ffff9e35da6f61c0 x1874672085349376/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827332 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827307.016748:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f61c0 x1874672085349376/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827332 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827307.016753:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827307.016756:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827307.016758:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085349376:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787827307.016761:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085349504:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787827307.016772:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827307.016784:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827307/real 1787827307] req@ffff9e35da6f5a40 x1874672085349504/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827332 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827307.016792:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5a40 x1874672085349504/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827332 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827307.016827:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827307.016830:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827307.016833:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085349504:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787827307.016837:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085349632:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787827307.016851:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827307.016858:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827307/real 1787827307] req@ffff9e35da6f7c00 x1874672085349632/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827332 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827307.016866:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7c00 x1874672085349632/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827332 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827307.016872:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827307.016874:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827307.016876:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085349632:192.168.203.141@tcp:8:kworker.0 00010000:00010000:3.0:1787827311.239947:0:6167:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827312.264025:0:33:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 16/16 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1787827315.335923:0:6167:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827319.431937:0:6167:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827322.503948:0:33:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 16/16 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1787827322.503978:0:6167:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 16 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1787827322.503986:0:6167:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9e35e0661800 lock: ffff9e35c2e59c00/0x55f45c2fc911dda4 lrc: 2/0,0 mode: PR/PR res: [0x200000402:0x1b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0xfc2f462b47d677b7 expref: -99 pid: 21497 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1787827322.504013:0:6167:0:(vvp_object.c:117:vvp_conf_set()) [0x200000402:0x1b:0x0]: losing layout lock 00010000:00010000:3.0:1787827322.504042:0:6167:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9e35e0661800 lock: ffff9e35c9584600/0x55f45c2fc911dc77 lrc: 2/0,0 mode: PR/PR res: [0x200000403:0x9:0x0].0x0 bits 0x20/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0xfc2f462b47d6746f expref: -99 pid: 20876 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1787827322.504064:0:6167:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9e35e0661800 lock: ffff9e35c87ea400/0x55f45c2fc911dc2a lrc: 2/0,0 mode: PR/PR res: [0x200000402:0x13:0x0].0x0 bits 0x1a/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0xfc2f462b47d673ab expref: -99 pid: 20295 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1787827322.504076:0:6167:0:(vvp_object.c:117:vvp_conf_set()) [0x200000402:0x13:0x0]: losing layout lock 00010000:00010000:3.0:1787827322.504080:0:6167:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9e35e0661800 lock: ffff9e35c343d800/0x55f45c2fc911daef lrc: 2/0,0 mode: PR/PR res: [0x200000403:0x2:0x0].0x0 bits 0x1a/0x0 rrc: 5 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0xfc2f462b47d67086 expref: -99 pid: 19561 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00000080:00200000:3.0:1787827322.504087:0:6167:0:(vvp_object.c:117:vvp_conf_set()) [0x200000403:0x2:0x0]: losing layout lock 00010000:00010000:3.0:1787827322.504091:0:6167:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9e35e0661800 lock: ffff9e35c9585800/0x55f45c2fc911dac5 lrc: 2/0,0 mode: PR/PR res: [0x200000403:0x2:0x0].0x0 bits 0x20/0x0 rrc: 5 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0xfc2f462b47d67016 expref: -99 pid: 19561 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1787827322.504101:0:6167:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9e35e0661800 lock: ffff9e35c932a600/0x55f45c2fc911dab7 lrc: 2/0,0 mode: CR/CR res: [0x200000403:0x2:0x0].0x0 bits 0x8/0x0 rrc: 5 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0xfc2f462b47d67001 expref: -99 pid: 19561 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1787827322.504105:0:6167:0:(vvp_object.c:117:vvp_conf_set()) [0x200000403:0x2:0x0]: losing layout lock 00010000:00010000:3.0:1787827322.504108:0:6167:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9e35e0661800 lock: ffff9e35c87eb600/0x55f45c2fc911da39 lrc: 2/0,0 mode: PR/PR res: [0x200000402:0xb:0x0].0x0 bits 0x12/0x0 rrc: 2 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0xfc2f462b47d66ea3 expref: -99 pid: 18548 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1787827322.504114:0:6167:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9e35e0661800 lock: ffff9e35c932b800/0x55f45c2fc911d998 lrc: 2/0,0 mode: CR/CR res: [0x200000402:0x9:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0xfc2f462b47d66c88 expref: -99 pid: 17393 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1787827322.504119:0:6167:0:(vvp_object.c:117:vvp_conf_set()) [0x200000402:0x9:0x0]: losing layout lock 00010000:00010000:3.0:1787827322.504123:0:6167:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9e35e0661800 lock: ffff9e35c87eb400/0x55f45c2fc911d96e lrc: 2/0,0 mode: PR/PR res: [0x200000402:0x8:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0xfc2f462b47d66c1f expref: -99 pid: 16818 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1787827322.504128:0:6167:0:(vvp_object.c:117:vvp_conf_set()) [0x200000402:0x8:0x0]: losing layout lock 00010000:00010000:3.0:1787827322.504136:0:6167:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9e35e0661800 lock: ffff9e35c9584a00/0x55f45c2fc911d952 lrc: 2/0,0 mode: PR/PR res: [0x200000402:0x7:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0xfc2f462b47d66bbd expref: -99 pid: 16241 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1787827322.504148:0:6167:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9e35e0661800 lock: ffff9e35c87ea200/0x55f45c2fc911d936 lrc: 2/0,0 mode: PR/PR res: [0x200000402:0x6:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0xfc2f462b47d66b69 expref: -99 pid: 15669 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1787827322.504158:0:6167:0:(vvp_object.c:117:vvp_conf_set()) [0x200000402:0x6:0x0]: losing layout lock 00010000:00010000:3.0:1787827322.504165:0:6167:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9e35e0661800 lock: ffff9e35c13f4c00/0x55f45c2fc911d737 lrc: 2/0,0 mode: PR/PR res: [0x200000403:0x1:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0xfc2f462b47d66503 expref: -99 pid: 14493 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1787827322.504171:0:6167:0:(vvp_object.c:117:vvp_conf_set()) [0x200000403:0x1:0x0]: losing layout lock 00010000:00010000:3.0:1787827322.504179:0:6167:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9e35e0661800 lock: ffff9e35c13f4800/0x55f45c2fc911d6e3 lrc: 2/0,0 mode: PR/PR res: [0x200000402:0x5:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0xfc2f462b47d66415 expref: -99 pid: 12759 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1787827322.504185:0:6167:0:(vvp_object.c:117:vvp_conf_set()) [0x200000402:0x5:0x0]: losing layout lock 00010000:00010000:3.0:1787827322.504191:0:6167:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9e35e0661800 lock: ffff9e35c0030c00/0x55f45c2fc911d729 lrc: 2/0,0 mode: PR/PR res: [0x200000402:0x4:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0xfc2f462b47d664d2 expref: -99 pid: 14488 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1787827322.504198:0:6167:0:(vvp_object.c:117:vvp_conf_set()) [0x200000402:0x4:0x0]: losing layout lock 00010000:00010000:3.0:1787827322.504205:0:6167:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9e35e0661800 lock: ffff9e35c0031e00/0x55f45c2fc911d722 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x20/0x0 rrc: 2 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0xfc2f462b47d664b6 expref: -99 pid: 14488 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1787827322.504215:0:6167:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9e35e0661800 lock: ffff9e35c1510a00/0x55f45c2fc911d68f lrc: 2/0,0 mode: CR/CR res: [0x200000402:0x4:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0xfc2f462b47d6632e expref: -99 pid: 12181 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1787827322.504220:0:6167:0:(vvp_object.c:117:vvp_conf_set()) [0x200000402:0x4:0x0]: losing layout lock 00010000:00010000:3.0:1787827322.504246:0:6167:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff9e35e0661800 lock: ffff9e35c2e59c00/0x55f45c2fc911dda4 lrc: 1/0,0 mode: --/PR res: [0x200000402:0x1b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xfc2f462b47d677b7 expref: -99 pid: 21497 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1787827322.504251:0:6167:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff9e35e0661800 lock: ffff9e35c9584600/0x55f45c2fc911dc77 lrc: 1/0,0 mode: --/PR res: [0x200000403:0x9:0x0].0x0 bits 0x20/0x0 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xfc2f462b47d6746f expref: -99 pid: 20876 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1787827322.504256:0:6167:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff9e35e0661800 lock: ffff9e35c87ea400/0x55f45c2fc911dc2a lrc: 1/0,0 mode: --/PR res: [0x200000402:0x13:0x0].0x0 bits 0x1a/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xfc2f462b47d673ab expref: -99 pid: 20295 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1787827322.504264:0:6167:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff9e35e0661800 lock: ffff9e35c343d800/0x55f45c2fc911daef lrc: 1/0,0 mode: --/PR res: [0x200000403:0x2:0x0].0x0 bits 0x1a/0x0 rrc: 5 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xfc2f462b47d67086 expref: -99 pid: 19561 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1787827322.504268:0:6167:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff9e35e0661800 lock: ffff9e35c9585800/0x55f45c2fc911dac5 lrc: 1/0,0 mode: --/PR res: [0x200000403:0x2:0x0].0x0 bits 0x20/0x0 rrc: 5 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xfc2f462b47d67016 expref: -99 pid: 19561 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1787827322.504272:0:6167:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff9e35e0661800 lock: ffff9e35c932a600/0x55f45c2fc911dab7 lrc: 1/0,0 mode: --/CR res: [0x200000403:0x2:0x0].0x0 bits 0x8/0x0 rrc: 5 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xfc2f462b47d67001 expref: -99 pid: 19561 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1787827322.504276:0:6167:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff9e35e0661800 lock: ffff9e35c87eb600/0x55f45c2fc911da39 lrc: 1/0,0 mode: --/PR res: [0x200000402:0xb:0x0].0x0 bits 0x12/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xfc2f462b47d66ea3 expref: -99 pid: 18548 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1787827322.504280:0:6167:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff9e35e0661800 lock: ffff9e35c932b800/0x55f45c2fc911d998 lrc: 1/0,0 mode: --/CR res: [0x200000402:0x9:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xfc2f462b47d66c88 expref: -99 pid: 17393 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1787827322.504284:0:6167:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff9e35e0661800 lock: ffff9e35c87eb400/0x55f45c2fc911d96e lrc: 1/0,0 mode: --/PR res: [0x200000402:0x8:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xfc2f462b47d66c1f expref: -99 pid: 16818 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1787827322.504288:0:6167:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff9e35e0661800 lock: ffff9e35c9584a00/0x55f45c2fc911d952 lrc: 1/0,0 mode: --/PR res: [0x200000402:0x7:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xfc2f462b47d66bbd expref: -99 pid: 16241 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1787827322.504292:0:6167:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff9e35e0661800 lock: ffff9e35c87ea200/0x55f45c2fc911d936 lrc: 1/0,0 mode: --/PR res: [0x200000402:0x6:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xfc2f462b47d66b69 expref: -99 pid: 15669 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1787827322.504296:0:6167:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff9e35e0661800 lock: ffff9e35c13f4c00/0x55f45c2fc911d737 lrc: 1/0,0 mode: --/PR res: [0x200000403:0x1:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xfc2f462b47d66503 expref: -99 pid: 14493 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1787827322.504300:0:6167:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff9e35e0661800 lock: ffff9e35c13f4800/0x55f45c2fc911d6e3 lrc: 1/0,0 mode: --/PR res: [0x200000402:0x5:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xfc2f462b47d66415 expref: -99 pid: 12759 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1787827322.504305:0:6167:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff9e35e0661800 lock: ffff9e35c0030c00/0x55f45c2fc911d729 lrc: 1/0,0 mode: --/PR res: [0x200000402:0x4:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xfc2f462b47d664d2 expref: -99 pid: 14488 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1787827322.504309:0:6167:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff9e35e0661800 lock: ffff9e35c0031e00/0x55f45c2fc911d722 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x20/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xfc2f462b47d664b6 expref: -99 pid: 14488 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1787827322.504313:0:6167:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff9e35e0661800 lock: ffff9e35c1510a00/0x55f45c2fc911d68f lrc: 1/0,0 mode: --/CR res: [0x200000402:0x4:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xfc2f462b47d6632e expref: -99 pid: 12181 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1787827322.504317:0:6167:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 16 locks packed 00010000:00010000:3.0:1787827322.504345:0:6167:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9e35e0661800 lock: ffff9e35c2e59c00/0x55f45c2fc911dda4 lrc: 0/0,0 mode: --/PR res: [0x200000402:0x1b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xfc2f462b47d677b7 expref: -99 pid: 21497 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1787827322.504358:0:6167:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9e35e0661800 lock: ffff9e35c9584600/0x55f45c2fc911dc77 lrc: 0/0,0 mode: --/PR res: [0x200000403:0x9:0x0].0x0 bits 0x20/0x0 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xfc2f462b47d6746f expref: -99 pid: 20876 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1787827322.504364:0:6167:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9e35e0661800 lock: ffff9e35c87ea400/0x55f45c2fc911dc2a lrc: 0/0,0 mode: --/PR res: [0x200000402:0x13:0x0].0x0 bits 0x1a/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xfc2f462b47d673ab expref: -99 pid: 20295 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1787827322.504374:0:6167:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9e35e0661800 lock: ffff9e35c343d800/0x55f45c2fc911daef lrc: 0/0,0 mode: --/PR res: [0x200000403:0x2:0x0].0x0 bits 0x1a/0x0 rrc: 5 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xfc2f462b47d67086 expref: -99 pid: 19561 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1787827322.504427:0:6167:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9e35e0661800 lock: ffff9e35c9585800/0x55f45c2fc911dac5 lrc: 0/0,0 mode: --/PR res: [0x200000403:0x2:0x0].0x0 bits 0x20/0x0 rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xfc2f462b47d67016 expref: -99 pid: 19561 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1787827322.504433:0:6167:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9e35e0661800 lock: ffff9e35c932a600/0x55f45c2fc911dab7 lrc: 0/0,0 mode: --/CR res: [0x200000403:0x2:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xfc2f462b47d67001 expref: -99 pid: 19561 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1787827322.504444:0:6167:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9e35e0661800 lock: ffff9e35c87eb600/0x55f45c2fc911da39 lrc: 0/0,0 mode: --/PR res: [0x200000402:0xb:0x0].0x0 bits 0x12/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xfc2f462b47d66ea3 expref: -99 pid: 18548 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1787827322.504451:0:6167:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9e35e0661800 lock: ffff9e35c932b800/0x55f45c2fc911d998 lrc: 0/0,0 mode: --/CR res: [0x200000402:0x9:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xfc2f462b47d66c88 expref: -99 pid: 17393 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1787827322.504463:0:6167:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9e35e0661800 lock: ffff9e35c87eb400/0x55f45c2fc911d96e lrc: 0/0,0 mode: --/PR res: [0x200000402:0x8:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xfc2f462b47d66c1f expref: -99 pid: 16818 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1787827322.504471:0:6167:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9e35e0661800 lock: ffff9e35c9584a00/0x55f45c2fc911d952 lrc: 0/0,0 mode: --/PR res: [0x200000402:0x7:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xfc2f462b47d66bbd expref: -99 pid: 16241 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1787827322.504478:0:6167:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9e35e0661800 lock: ffff9e35c87ea200/0x55f45c2fc911d936 lrc: 0/0,0 mode: --/PR res: [0x200000402:0x6:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xfc2f462b47d66b69 expref: -99 pid: 15669 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1787827322.504485:0:6167:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9e35e0661800 lock: ffff9e35c13f4c00/0x55f45c2fc911d737 lrc: 0/0,0 mode: --/PR res: [0x200000403:0x1:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xfc2f462b47d66503 expref: -99 pid: 14493 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1787827322.504492:0:6167:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9e35e0661800 lock: ffff9e35c13f4800/0x55f45c2fc911d6e3 lrc: 0/0,0 mode: --/PR res: [0x200000402:0x5:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xfc2f462b47d66415 expref: -99 pid: 12759 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1787827322.504499:0:6167:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9e35e0661800 lock: ffff9e35c0030c00/0x55f45c2fc911d729 lrc: 0/0,0 mode: --/PR res: [0x200000402:0x4:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xfc2f462b47d664d2 expref: -99 pid: 14488 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1787827322.504509:0:6167:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9e35e0661800 lock: ffff9e35c0031e00/0x55f45c2fc911d722 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x20/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xfc2f462b47d664b6 expref: -99 pid: 14488 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1787827322.504520:0:6167:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9e35e0661800 lock: ffff9e35c1510a00/0x55f45c2fc911d68f lrc: 0/0,0 mode: --/CR res: [0x200000402:0x4:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xfc2f462b47d6632e expref: -99 pid: 12181 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1787827322.504529:0:6167:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827322.504530:0:6167:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827322.504532:0:6167:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1787827325.448129:0:2399:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.141:988 00000400:00000100:3.0:1787827325.448177:0:2399:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.141@tcp at host 192.168.203.141 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1787827325.448195:0:2399:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1787827325.448378:0:2399:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000800:00000100:3.0:1787827325.448418:0:2399:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1787827325.448437:0:2399:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1787827325.448461:0:2399:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1787827325.448477:0:2399:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1787827325.448494:0:2399:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1787827325.448512:0:2399:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1787827325.448527:0:2399:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00010000:00010000:3.0:1787827326.599975:0:6167:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827328.647944:0:33:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 8/8 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1787827329.672022:0:6167:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827333.767903:0:6167:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827334.791941:0:33:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 3/3 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1787827336.839926:0:6167:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827340.935901:0:6167:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827342.856082:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085357312:192.168.203.141@tcp:250:kworker.0 00000800:00000100:3.0:1787827342.856145:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827342.856196:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827342/real 1787827342] req@ffff9e35da7192c0 x1874672085357312/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827367 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827342.856231:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7192c0 x1874672085357312/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827367 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827342.856248:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827342.856253:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827342.856258:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085357312:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1787827342.856287:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085357440:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787827342.856317:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827342.856333:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827342/real 1787827342] req@ffff9e35da71a580 x1874672085357440/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827367 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827342.856353:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672085357440/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827367 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827342.856380:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827342.856385:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827342.856389:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085357440:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787827342.856394:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085357568:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787827342.856408:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827342.856416:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827342/real 1787827342] req@ffff9e35da718780 x1874672085357568/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827367 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827342.856424:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718780 x1874672085357568/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827367 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827342.856433:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827342.856436:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827342.856438:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085357568:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787827342.856442:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085357696:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787827342.856452:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827342.856463:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827342/real 1787827342] req@ffff9e35da718000 x1874672085357696/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827367 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827342.856475:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718000 x1874672085357696/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827367 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827342.856496:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827342.856499:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827342.856503:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085357696:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787827342.856510:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085357824:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787827342.856528:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827342.856537:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827342/real 1787827342] req@ffff9e35da719a40 x1874672085357824/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827367 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827342.856544:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672085357824/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827367 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827342.856550:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827342.856552:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827342.856555:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085357824:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787827342.856558:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085357952:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787827342.856569:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827342.856575:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827342/real 1787827342] req@ffff9e35da71a1c0 x1874672085357952/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827367 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827342.856582:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a1c0 x1874672085357952/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827367 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827342.856587:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827342.856589:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827342.856591:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085357952:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787827342.856594:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085358080:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787827342.856603:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827342.856609:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827342/real 1787827342] req@ffff9e35da71a940 x1874672085358080/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827367 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827342.856627:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672085358080/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827367 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827342.856632:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827342.856634:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827342.856636:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085358080:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787827342.856649:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085358208:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787827342.856660:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827342.856667:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827342/real 1787827342] req@ffff9e35da718f00 x1874672085358208/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827367 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827342.856674:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672085358208/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827367 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827342.856679:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827342.856681:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827342.856683:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085358208:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787827342.856698:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085358336:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787827342.856707:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827342.856713:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827342/real 1787827342] req@ffff9e35da718b40 x1874672085358336/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827367 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827342.856720:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718b40 x1874672085358336/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827367 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827342.856734:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827342.856737:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827342.856752:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085358336:192.168.203.141@tcp:8:kworker.0 00010000:00010000:3.0:1787827345.031952:0:6167:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827347.976050:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085358464:192.168.203.141@tcp:250:kworker.0 00000800:00000100:3.0:1787827347.976115:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827347.976174:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827347/real 1787827347] req@ffff9e35da7183c0 x1874672085358464/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827372 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827347.976204:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672085358464/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827372 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827347.976230:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827347.976239:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827347.976313:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085358464:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1787827347.976341:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085358592:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787827347.976508:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827347.976524:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827347/real 1787827347] req@ffff9e35da71bc00 x1874672085358592/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827372 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827347.976539:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71bc00 x1874672085358592/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827372 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827347.976546:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827347.976550:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827347.976552:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085358592:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787827347.976559:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085358720:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787827347.976574:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827347.976582:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827347/real 1787827347] req@ffff9e35da71b480 x1874672085358720/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827372 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827347.976589:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71b480 x1874672085358720/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827372 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827347.976594:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827347.976596:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827347.976598:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085358720:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787827347.976602:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085358848:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787827347.976613:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827347.976619:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827347/real 1787827347] req@ffff9e35da718b40 x1874672085358848/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827372 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827347.976626:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718b40 x1874672085358848/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827372 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827347.976631:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827347.976633:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827347.976635:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085358848:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787827347.976639:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085358976:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787827347.976648:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827347.976654:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827347/real 1787827347] req@ffff9e35da718f00 x1874672085358976/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827372 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827347.976661:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672085358976/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827372 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827347.976666:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827347.976668:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827347.976669:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085358976:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787827347.976673:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085359104:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787827347.976683:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827347.976689:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827347/real 1787827347] req@ffff9e35da71a940 x1874672085359104/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827372 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827347.976697:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672085359104/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827372 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827347.976702:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827347.976721:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827347.976723:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085359104:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787827347.976727:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085359232:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787827347.976742:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827347.976749:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827347/real 1787827347] req@ffff9e35da71a1c0 x1874672085359232/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827372 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827347.976756:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a1c0 x1874672085359232/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827372 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827347.976762:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827347.976764:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827347.976766:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085359232:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787827347.976769:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085359360:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787827347.976779:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827347.976784:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827347/real 1787827347] req@ffff9e35da719a40 x1874672085359360/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827372 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827347.976791:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672085359360/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827372 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827347.976821:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827347.976825:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827347.976827:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085359360:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787827347.976849:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085359488:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787827347.976862:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827347.976871:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827347/real 1787827347] req@ffff9e35da718000 x1874672085359488/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827372 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827347.976880:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718000 x1874672085359488/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827372 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827347.976887:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827347.976890:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827347.976893:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085359488:192.168.203.141@tcp:8:kworker.0 00010000:00010000:3.0:1787827349.127943:0:6167:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827352.199953:0:33:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 16/16 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1787827352.199982:0:6167:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 16 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1787827352.199989:0:6167:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9e35d0749800 lock: ffff9e35c4918a00/0x55f45c2fc9174749 lrc: 2/0,0 mode: PR/PR res: [0x200000402:0x23:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0xfc2f462b47f418a0 expref: -99 pid: 25074 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1787827352.200019:0:6167:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9e35d0749800 lock: ffff9e35c4919200/0x55f45c2fc9174750 lrc: 2/0,0 mode: PR/PR res: [0x200000400:0x6:0x0].0x0 bits 0x12/0x0 rrc: 2 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0xfc2f462b47f418ae expref: -99 pid: 25074 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1787827352.200033:0:6167:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9e35d0749800 lock: ffff9e35c2e58400/0x55f45c2fc9174711 lrc: 2/0,0 mode: PR/PR res: [0x200000402:0x22:0x0].0x0 bits 0x20/0x0 rrc: 4 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0xfc2f462b47f417d5 expref: -99 pid: 24485 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1787827352.200056:0:6167:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9e35d0749800 lock: ffff9e35c4919c00/0x55f45c2fc917470a lrc: 2/0,0 mode: PR/PR res: [0x200000402:0x22:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0xfc2f462b47f417c0 expref: -99 pid: 24485 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1787827352.200067:0:6167:0:(vvp_object.c:117:vvp_conf_set()) [0x200000402:0x22:0x0]: losing layout lock 00010000:00010000:3.0:1787827352.200089:0:6167:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9e35d0749800 lock: ffff9e35c87ebe00/0x55f45c2fc9174703 lrc: 2/0,0 mode: CR/CR res: [0x200000402:0x22:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0xfc2f462b47f41796 expref: -99 pid: 24484 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1787827352.200094:0:6167:0:(vvp_object.c:117:vvp_conf_set()) [0x200000402:0x22:0x0]: losing layout lock 00010000:00010000:3.0:1787827352.200098:0:6167:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9e35d0749800 lock: ffff9e35c4919400/0x55f45c2fc91746d2 lrc: 2/0,0 mode: PR/PR res: [0x200000402:0x21:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0xfc2f462b47f41726 expref: -99 pid: 24484 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1787827352.200118:0:6167:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9e35d0749800 lock: ffff9e35c932be00/0x55f45c2fc9174669 lrc: 2/0,0 mode: PR/PR res: [0x200000402:0x1f:0x0].0x0 bits 0x20/0x0 rrc: 4 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0xfc2f462b47f41607 expref: -99 pid: 23894 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1787827352.200127:0:6167:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9e35d0749800 lock: ffff9e35c932a200/0x55f45c2fc9174662 lrc: 2/0,0 mode: PR/PR res: [0x200000402:0x1f:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0xfc2f462b47f415f2 expref: -99 pid: 23894 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1787827352.200133:0:6167:0:(vvp_object.c:117:vvp_conf_set()) [0x200000402:0x1f:0x0]: losing layout lock 00010000:00010000:3.0:1787827352.200141:0:6167:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9e35d0749800 lock: ffff9e35c932a000/0x55f45c2fc917465b lrc: 2/0,0 mode: CR/CR res: [0x200000402:0x1f:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0xfc2f462b47f415c8 expref: -99 pid: 23892 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1787827352.200145:0:6167:0:(vvp_object.c:117:vvp_conf_set()) [0x200000402:0x1f:0x0]: losing layout lock 00010000:00010000:3.0:1787827352.200149:0:6167:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9e35d0749800 lock: ffff9e35c932a800/0x55f45c2fc917462a lrc: 2/0,0 mode: PR/PR res: [0x200000402:0x1e:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0xfc2f462b47f4154a expref: -99 pid: 23892 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1787827352.200158:0:6167:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9e35d0749800 lock: ffff9e35c2e59200/0x55f45c2fc9173d6a lrc: 2/0,0 mode: PR/PR res: [0x240000402:0x7:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0xfc2f462b47f3fb9d expref: -99 pid: 23142 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1787827352.200168:0:6167:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9e35d0749800 lock: ffff9e35c1510e00/0x55f45c2fc911ddc0 lrc: 2/0,0 mode: PR/PR res: [0x240000402:0x4:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0xfc2f462b47d6783c expref: -99 pid: 22087 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1787827352.200179:0:6167:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9e35d0749800 lock: ffff9e35c1511c00/0x55f45c2fc911ddd5 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x4:0x0].0x0 bits 0x12/0x0 rrc: 2 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0xfc2f462b47d67866 expref: -99 pid: 22087 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1787827352.200186:0:6167:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9e35d0749800 lock: ffff9e35c9585a00/0x55f45c2fc911dd96 lrc: 2/0,0 mode: CR/CR res: [0x200000402:0x1b:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0xfc2f462b47d6778d expref: -99 pid: 21495 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1787827352.200192:0:6167:0:(vvp_object.c:117:vvp_conf_set()) [0x200000402:0x1b:0x0]: losing layout lock 00010000:00010000:3.0:1787827352.200195:0:6167:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9e35d0749800 lock: ffff9e35c87eae00/0x55f45c2fc911dd6c lrc: 2/0,0 mode: PR/PR res: [0x200000403:0x9:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0xfc2f462b47d67732 expref: -99 pid: 20918 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1787827352.200201:0:6167:0:(vvp_object.c:117:vvp_conf_set()) [0x200000403:0x9:0x0]: losing layout lock 00010000:00010000:3.0:1787827352.200208:0:6167:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9e35d0749800 lock: ffff9e35c343ce00/0x55f45c2fc911dc5b lrc: 2/0,0 mode: CR/CR res: [0x200000403:0x9:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0xfc2f462b47d67429 expref: -99 pid: 20868 timeout: 0 lvb_type: 3 lru_score: 7 lru_type: 0 00000080:00200000:3.0:1787827352.200213:0:6167:0:(vvp_object.c:117:vvp_conf_set()) [0x200000403:0x9:0x0]: losing layout lock 00010000:00010000:3.0:1787827352.200252:0:6167:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff9e35d0749800 lock: ffff9e35c4918a00/0x55f45c2fc9174749 lrc: 1/0,0 mode: --/PR res: [0x200000402:0x23:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xfc2f462b47f418a0 expref: -99 pid: 25074 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1787827352.200261:0:6167:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff9e35d0749800 lock: ffff9e35c4919200/0x55f45c2fc9174750 lrc: 1/0,0 mode: --/PR res: [0x200000400:0x6:0x0].0x0 bits 0x12/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xfc2f462b47f418ae expref: -99 pid: 25074 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1787827352.200268:0:6167:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff9e35d0749800 lock: ffff9e35c2e58400/0x55f45c2fc9174711 lrc: 1/0,0 mode: --/PR res: [0x200000402:0x22:0x0].0x0 bits 0x20/0x0 rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xfc2f462b47f417d5 expref: -99 pid: 24485 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1787827352.200275:0:6167:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff9e35d0749800 lock: ffff9e35c4919c00/0x55f45c2fc917470a lrc: 1/0,0 mode: --/PR res: [0x200000402:0x22:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xfc2f462b47f417c0 expref: -99 pid: 24485 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1787827352.200283:0:6167:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff9e35d0749800 lock: ffff9e35c87ebe00/0x55f45c2fc9174703 lrc: 1/0,0 mode: --/CR res: [0x200000402:0x22:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xfc2f462b47f41796 expref: -99 pid: 24484 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1787827352.200291:0:6167:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff9e35d0749800 lock: ffff9e35c4919400/0x55f45c2fc91746d2 lrc: 1/0,0 mode: --/PR res: [0x200000402:0x21:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xfc2f462b47f41726 expref: -99 pid: 24484 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1787827352.200299:0:6167:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff9e35d0749800 lock: ffff9e35c932be00/0x55f45c2fc9174669 lrc: 1/0,0 mode: --/PR res: [0x200000402:0x1f:0x0].0x0 bits 0x20/0x0 rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xfc2f462b47f41607 expref: -99 pid: 23894 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1787827352.200307:0:6167:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff9e35d0749800 lock: ffff9e35c932a200/0x55f45c2fc9174662 lrc: 1/0,0 mode: --/PR res: [0x200000402:0x1f:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xfc2f462b47f415f2 expref: -99 pid: 23894 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1787827352.200312:0:6167:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff9e35d0749800 lock: ffff9e35c932a000/0x55f45c2fc917465b lrc: 1/0,0 mode: --/CR res: [0x200000402:0x1f:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xfc2f462b47f415c8 expref: -99 pid: 23892 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1787827352.200316:0:6167:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff9e35d0749800 lock: ffff9e35c932a800/0x55f45c2fc917462a lrc: 1/0,0 mode: --/PR res: [0x200000402:0x1e:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xfc2f462b47f4154a expref: -99 pid: 23892 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1787827352.200320:0:6167:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff9e35d0749800 lock: ffff9e35c2e59200/0x55f45c2fc9173d6a lrc: 1/0,0 mode: --/PR res: [0x240000402:0x7:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xfc2f462b47f3fb9d expref: -99 pid: 23142 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1787827352.200324:0:6167:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff9e35d0749800 lock: ffff9e35c1510e00/0x55f45c2fc911ddc0 lrc: 1/0,0 mode: --/PR res: [0x240000402:0x4:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xfc2f462b47d6783c expref: -99 pid: 22087 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1787827352.200328:0:6167:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff9e35d0749800 lock: ffff9e35c1511c00/0x55f45c2fc911ddd5 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x4:0x0].0x0 bits 0x12/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xfc2f462b47d67866 expref: -99 pid: 22087 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1787827352.200332:0:6167:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff9e35d0749800 lock: ffff9e35c9585a00/0x55f45c2fc911dd96 lrc: 1/0,0 mode: --/CR res: [0x200000402:0x1b:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xfc2f462b47d6778d expref: -99 pid: 21495 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1787827352.200338:0:6167:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff9e35d0749800 lock: ffff9e35c87eae00/0x55f45c2fc911dd6c lrc: 1/0,0 mode: --/PR res: [0x200000403:0x9:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xfc2f462b47d67732 expref: -99 pid: 20918 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1787827352.200342:0:6167:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff9e35d0749800 lock: ffff9e35c343ce00/0x55f45c2fc911dc5b lrc: 1/0,0 mode: --/CR res: [0x200000403:0x9:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xfc2f462b47d67429 expref: -99 pid: 20868 timeout: 0 lvb_type: 3 lru_score: 7 lru_type: 0 00010000:00010000:3.0:1787827352.200346:0:6167:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 16 locks packed 00010000:00010000:3.0:1787827352.200370:0:6167:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9e35d0749800 lock: ffff9e35c4918a00/0x55f45c2fc9174749 lrc: 0/0,0 mode: --/PR res: [0x200000402:0x23:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xfc2f462b47f418a0 expref: -99 pid: 25074 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1787827352.200382:0:6167:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9e35d0749800 lock: ffff9e35c4919200/0x55f45c2fc9174750 lrc: 0/0,0 mode: --/PR res: [0x200000400:0x6:0x0].0x0 bits 0x12/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xfc2f462b47f418ae expref: -99 pid: 25074 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1787827352.200388:0:6167:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9e35d0749800 lock: ffff9e35c2e58400/0x55f45c2fc9174711 lrc: 0/0,0 mode: --/PR res: [0x200000402:0x22:0x0].0x0 bits 0x20/0x0 rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xfc2f462b47f417d5 expref: -99 pid: 24485 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1787827352.200394:0:6167:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9e35d0749800 lock: ffff9e35c4919c00/0x55f45c2fc917470a lrc: 0/0,0 mode: --/PR res: [0x200000402:0x22:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xfc2f462b47f417c0 expref: -99 pid: 24485 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1787827352.200400:0:6167:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9e35d0749800 lock: ffff9e35c87ebe00/0x55f45c2fc9174703 lrc: 0/0,0 mode: --/CR res: [0x200000402:0x22:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xfc2f462b47f41796 expref: -99 pid: 24484 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1787827352.200407:0:6167:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9e35d0749800 lock: ffff9e35c4919400/0x55f45c2fc91746d2 lrc: 0/0,0 mode: --/PR res: [0x200000402:0x21:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xfc2f462b47f41726 expref: -99 pid: 24484 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1787827352.200415:0:6167:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9e35d0749800 lock: ffff9e35c932be00/0x55f45c2fc9174669 lrc: 0/0,0 mode: --/PR res: [0x200000402:0x1f:0x0].0x0 bits 0x20/0x0 rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xfc2f462b47f41607 expref: -99 pid: 23894 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1787827352.200427:0:6167:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9e35d0749800 lock: ffff9e35c932a200/0x55f45c2fc9174662 lrc: 0/0,0 mode: --/PR res: [0x200000402:0x1f:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xfc2f462b47f415f2 expref: -99 pid: 23894 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1787827352.200444:0:6167:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9e35d0749800 lock: ffff9e35c932a000/0x55f45c2fc917465b lrc: 0/0,0 mode: --/CR res: [0x200000402:0x1f:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xfc2f462b47f415c8 expref: -99 pid: 23892 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1787827352.200455:0:6167:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9e35d0749800 lock: ffff9e35c932a800/0x55f45c2fc917462a lrc: 0/0,0 mode: --/PR res: [0x200000402:0x1e:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xfc2f462b47f4154a expref: -99 pid: 23892 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1787827352.200463:0:6167:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9e35d0749800 lock: ffff9e35c2e59200/0x55f45c2fc9173d6a lrc: 0/0,0 mode: --/PR res: [0x240000402:0x7:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xfc2f462b47f3fb9d expref: -99 pid: 23142 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1787827352.200473:0:6167:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9e35d0749800 lock: ffff9e35c1510e00/0x55f45c2fc911ddc0 lrc: 0/0,0 mode: --/PR res: [0x240000402:0x4:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xfc2f462b47d6783c expref: -99 pid: 22087 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1787827352.200481:0:6167:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9e35d0749800 lock: ffff9e35c1511c00/0x55f45c2fc911ddd5 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x4:0x0].0x0 bits 0x12/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xfc2f462b47d67866 expref: -99 pid: 22087 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1787827352.200498:0:6167:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9e35d0749800 lock: ffff9e35c9585a00/0x55f45c2fc911dd96 lrc: 0/0,0 mode: --/CR res: [0x200000402:0x1b:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xfc2f462b47d6778d expref: -99 pid: 21495 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1787827352.200505:0:6167:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9e35d0749800 lock: ffff9e35c87eae00/0x55f45c2fc911dd6c lrc: 0/0,0 mode: --/PR res: [0x200000403:0x9:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xfc2f462b47d67732 expref: -99 pid: 20918 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1787827352.200511:0:6167:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9e35d0749800 lock: ffff9e35c343ce00/0x55f45c2fc911dc5b lrc: 0/0,0 mode: --/CR res: [0x200000403:0x9:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xfc2f462b47d67429 expref: -99 pid: 20868 timeout: 0 lvb_type: 3 lru_score: 7 lru_type: 0 00010000:00010000:3.0:1787827352.200522:0:6167:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827352.200523:0:6167:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827352.200524:0:6167:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827353.096003:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085359744:192.168.203.141@tcp:250:kworker.0 00000800:00000100:3.0:1787827353.096062:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827353.096100:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827353/real 1787827353] req@ffff9e35da718780 x1874672085359744/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827378 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827353.096125:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718780 x1874672085359744/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827378 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827353.096148:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827353.096155:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827353.096164:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085359744:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1787827353.096196:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085359872:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787827353.096220:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827353.096235:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827353/real 1787827353] req@ffff9e35da71a580 x1874672085359872/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827378 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827353.096253:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672085359872/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827378 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827353.096262:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827353.096266:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827353.096270:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085359872:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787827353.096288:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085360000:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787827353.096314:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827353.096332:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827353/real 1787827353] req@ffff9e35da7192c0 x1874672085360000/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827378 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827353.096365:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7192c0 x1874672085360000/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827378 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827353.096371:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827353.096374:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827353.096376:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085360000:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787827353.096381:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085360128:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787827353.096396:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827353.096403:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827353/real 1787827353] req@ffff9e35da718000 x1874672085360128/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827378 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827353.096411:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718000 x1874672085360128/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827378 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827353.096416:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827353.096418:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827353.096420:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085360128:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787827353.096423:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085360256:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787827353.096433:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827353.096438:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827353/real 1787827353] req@ffff9e35da719a40 x1874672085360256/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827378 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827353.096446:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672085360256/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827378 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827353.096451:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827353.096452:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827353.096454:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085360256:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787827353.096457:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085360384:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787827353.096467:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827353.096473:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827353/real 1787827353] req@ffff9e35da71a1c0 x1874672085360384/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827378 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827353.096480:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a1c0 x1874672085360384/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827378 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827353.096485:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827353.096487:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827353.096488:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085360384:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787827353.096501:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085360512:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787827353.096511:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827353.096517:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827353/real 1787827353] req@ffff9e35da71a940 x1874672085360512/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827378 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827353.096524:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672085360512/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827378 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827353.096534:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827353.096536:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827353.096538:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085360512:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787827353.096541:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085360640:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787827353.096550:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827353.096556:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827353/real 1787827353] req@ffff9e35da718f00 x1874672085360640/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827378 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827353.096563:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672085360640/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827378 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827353.096568:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827353.096570:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827353.096572:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085360640:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787827353.096575:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085360768:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787827353.096584:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827353.096589:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827353/real 1787827353] req@ffff9e35da718b40 x1874672085360768/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827378 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827353.096596:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718b40 x1874672085360768/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827378 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827353.096601:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827353.096602:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827353.096605:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085360768:192.168.203.141@tcp:8:kworker.0 00010000:00010000:3.0:1787827355.271952:0:33:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 4/4 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1787827355.271969:0:33:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 2/6 locks to regular blp list, 2 blwis in pool 00010000:00010000:3.0:1787827355.271988:0:33:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 2/8 locks to regular blp list, 3 blwis in pool 00010000:00010000:3.0:1787827355.272043:0:33:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 16/18 locks to regular blp list, 2 blwis in pool 00010000:00010000:3.0:1787827355.272080:0:6167:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 16 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1787827355.272407:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827355.272420:0:6167:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff9e35d0749800 lock: ffff9e35c2fa6a00/0x55f45c2fc91748a0 lrc: 2/0,0 mode: PR/PR res: [0x240000402:0xb:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0xfc2f462b47f41cac expref: -99 pid: 26405 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1787827355.272469:0:6167:0:(vvp_object.c:117:vvp_conf_set()) [0x240000402:0xb:0x0]: losing layout lock 00010000:00010000:3.0:1787827355.272511:0:6167:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff9e35d0749800 lock: ffff9e35c2fa7000/0x55f45c2fc917488b lrc: 2/0,0 mode: CR/CR res: [0x240000402:0xb:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0xfc2f462b47f41c4a expref: -99 pid: 26262 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1787827355.272522:0:6167:0:(vvp_object.c:117:vvp_conf_set()) [0x240000402:0xb:0x0]: losing layout lock 00010000:00010000:3.0:1787827355.272529:0:6167:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff9e35d0749800 lock: ffff9e35c2fa7c00/0x55f45c2fc9174899 lrc: 2/0,0 mode: PR/PR res: [0x240000402:0xb:0x0].0x0 bits 0x20/0x0 rrc: 4 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0xfc2f462b47f41c89 expref: -99 pid: 26263 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1787827355.272544:0:6167:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff9e35d0749800 lock: ffff9e35c2fa7400/0x55f45c2fc91747d5 lrc: 2/0,0 mode: PR/PR res: [0x240000401:0x7:0x0].0x0 bits 0x12/0x0 rrc: 2 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0xfc2f462b47f41a4b expref: -99 pid: 25667 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1787827355.272556:0:6167:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff9e35d0749800 lock: ffff9e35c13f5600/0x55f45c2fc91747b2 lrc: 2/0,0 mode: CR/CR res: [0x240000402:0xa:0x0].0x0 bits 0xd/0x0 rrc: 5 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0xfc2f462b47f419bf expref: -99 pid: 25080 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000002:00100000:3.0:1787827355.272598:0:6167:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff9e35da773480 x1874672085139968/t4294967365(4294967365) o101->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 576/664 e 0 to 0 dl 1787826923 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'cmp.0' uid:0 gid:0 projid:0 00000020:00100000:3.0:1787827355.272607:0:6167:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0001-mdc-ffff9e35d0749800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:3.0:1787827355.272611:0:6167:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9e35da7703c0 x1874672085361408/t0(0) o35->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:23/10 lens 392/624 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00080000:3.0:1787827355.272624:0:6167:0:(client.c:1845:ptlrpc_send_new_req()) @@@ req waiting for recovery: (FULL != DISCONN) req@ffff9e35da7703c0 x1874672085361408/t0(0) o35->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:23/10 lens 392/624 e 0 to 0 dl 0 ref 2 fl Rpc:WQU/600/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:0 00000100:00100000:3.0:1787827355.272629:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787827358.215866:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827358.215876:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787827358.215881:0:8:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827358.215891:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 793 00000100:00080000:3.0:1787827358.215903:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827358.215910:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787827358.215946:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da771a40 x1874672085361536/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827358.215978:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827358.215981:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827358.215984:0:8:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827358.215988:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 793 00000100:00080000:3.0:1787827358.215992:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827358.215996:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827358.216008:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827358.216022:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7730c0 x1874672085361664/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827358.216032:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827358.216035:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787827358.216036:0:8:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827358.216039:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 793 00000100:00080000:3.0:1787827358.216042:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827358.216045:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827358.216049:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827358.216058:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da770000 x1874672085361792/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827358.216064:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827358.216067:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787827358.216068:0:8:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827358.216070:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 793 00000100:00080000:3.0:1787827358.216073:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827358.216080:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827358.216085:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:3.0:1787827358.216093:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da772940 x1874672085361920/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827358.216100:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827358.216103:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787827358.216104:0:8:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827358.216109:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 793 00000100:00080000:3.0:1787827358.216114:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827358.216123:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827358.216132:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:3.0:1787827358.216151:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da771e00 x1874672085362048/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827358.216169:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827358.216173:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827358.216175:0:8:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827358.216180:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 793 00000100:00080000:3.0:1787827358.216188:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827358.216192:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827358.216196:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827358.216210:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da770f00 x1874672085362176/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827358.216223:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827358.216229:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787827358.216231:0:8:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827358.216237:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 793 00000100:00080000:3.0:1787827358.216245:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827358.216253:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827358.216264:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827358.216285:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da771a40 x1874672085362304/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827358.216302:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827358.216307:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787827358.216311:0:8:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827358.216323:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 793 00000100:00080000:3.0:1787827358.216334:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827358.216345:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827358.216358:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:3.0:1787827358.216376:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7712c0 x1874672085362432/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827358.216387:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827358.216396:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787827358.216400:0:8:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827358.216407:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 793 00000100:00080000:3.0:1787827358.216414:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827358.216420:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827358.216428:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:3.0:1787827358.216442:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da772580 x1874672085362560/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:3.0:1787827362.439930:0:33:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 14/14 locks to regular blp list, 1 blwis in pool 00000100:00100000:3.0:1787827369.607940:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787827374.728045:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827378.824070:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827382.920025:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827387.015930:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827388.936239:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085368576:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1787827388.936435:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085368704:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787827388.936492:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085368832:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787827388.936549:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085368960:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787827388.936591:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085369088:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787827388.936627:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085369216:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787827388.936679:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085369344:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787827388.936725:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085369472:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787827388.936775:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085369600:192.168.203.141@tcp:8:kworker.0 00010000:00010000:3.0:1787827391.112029:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827392.009203:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827388/real 1787827392] req@ffff9e35da71a580 x1874672085369600/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827413 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827392.009247:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672085369600/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827413 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827392.009268:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827392.009277:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827392.009285:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085369600:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787827392.009299:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827388/real 1787827392] req@ffff9e35da7192c0 x1874672085369472/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827413 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827392.009321:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7192c0 x1874672085369472/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827413 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827392.009336:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827392.009344:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827392.009350:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085369472:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787827392.009361:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827388/real 1787827392] req@ffff9e35da718000 x1874672085369344/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827413 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827392.009383:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718000 x1874672085369344/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827413 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827392.009414:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827392.009422:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827392.009429:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085369344:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787827392.009440:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827388/real 1787827392] req@ffff9e35da71a1c0 x1874672085369088/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827413 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827392.009461:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a1c0 x1874672085369088/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827413 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827392.009476:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827392.009482:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827392.009488:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085369088:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787827392.009498:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827388/real 1787827392] req@ffff9e35da719a40 x1874672085369216/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827413 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827392.009520:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672085369216/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827413 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827392.009534:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827392.009541:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827392.009546:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085369216:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787827392.009557:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827388/real 1787827392] req@ffff9e35da718b40 x1874672085368704/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827413 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827392.009579:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718b40 x1874672085368704/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827413 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827392.009601:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827392.009608:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827392.009619:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085368704:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787827392.009630:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827388/real 1787827392] req@ffff9e35da718f00 x1874672085368832/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827413 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827392.009653:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672085368832/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827413 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827392.009667:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827392.010225:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827392.010234:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085368832:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787827392.010247:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827388/real 1787827392] req@ffff9e35da71a940 x1874672085368960/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827413 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827392.010272:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672085368960/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827413 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827392.010287:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827392.010293:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827392.010300:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085368960:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787827392.010311:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827388/real 1787827392] req@ffff9e35da7183c0 x1874672085368576/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827413 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827392.010333:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672085368576/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827413 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827392.010348:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827392.010354:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827392.010360:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085368576:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1787827394.056778:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085369728:192.168.203.141@tcp:250:kworker.0 00000800:00000100:3.0:1787827394.056939:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827394.056986:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827394/real 1787827394] req@ffff9e35da718780 x1874672085369728/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827419 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827394.057012:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718780 x1874672085369728/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827419 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827394.057028:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827394.057034:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827394.057039:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085369728:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1787827394.057051:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085369856:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787827394.057085:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827394.057111:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827394/real 1787827394] req@ffff9e35da7183c0 x1874672085369856/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827419 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827394.057136:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672085369856/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827419 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827394.057147:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827394.057152:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827394.057157:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085369856:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787827394.057166:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085369984:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787827394.057205:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827394.057219:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827394/real 1787827394] req@ffff9e35da71a940 x1874672085369984/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827419 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827394.057239:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672085369984/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827419 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827394.057245:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827394.057248:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827394.057250:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085369984:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787827394.057256:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085370112:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787827394.057273:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827394.057281:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827394/real 1787827394] req@ffff9e35da718f00 x1874672085370112/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827419 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827394.057289:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672085370112/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827419 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827394.057305:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827394.057308:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827394.057310:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085370112:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787827394.057315:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085370240:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787827394.057327:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827394.057334:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827394/real 1787827394] req@ffff9e35da718b40 x1874672085370240/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827419 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827394.057342:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718b40 x1874672085370240/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827419 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827394.057348:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827394.057350:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827394.057352:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085370240:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787827394.057357:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085370368:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787827394.057368:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827394.057375:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827394/real 1787827394] req@ffff9e35da719a40 x1874672085370368/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827419 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827394.057406:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672085370368/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827419 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827394.057420:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827394.057424:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827394.057428:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085370368:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787827394.057436:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085370496:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787827394.057471:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827394.057485:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827394/real 1787827394] req@ffff9e35da71a1c0 x1874672085370496/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827419 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827394.057498:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a1c0 x1874672085370496/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827419 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827394.057508:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827394.057511:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827394.057541:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085370496:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787827394.057550:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085370624:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787827394.057573:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827394.057591:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827394/real 1787827394] req@ffff9e35da718000 x1874672085370624/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827419 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827394.057617:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718000 x1874672085370624/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827419 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827394.057626:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827394.057629:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827394.057633:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085370624:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787827394.057641:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085370752:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787827394.057665:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827394.057678:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827394/real 1787827394] req@ffff9e35da7192c0 x1874672085370752/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827419 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827394.057690:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7192c0 x1874672085370752/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827419 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827394.057706:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827394.057710:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827394.057713:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085370752:192.168.203.141@tcp:8:kworker.0 00010000:00010000:3.0:1787827394.183979:0:15664:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827395.208009:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827398.279944:0:15664:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827399.177336:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085370880:192.168.203.141@tcp:250:kworker.0 00000800:00000100:3.0:1787827399.177412:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827399.177453:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827399/real 1787827399] req@ffff9e35da71a580 x1874672085370880/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827424 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827399.177465:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672085370880/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827424 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827399.177473:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827399.177476:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827399.177479:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085370880:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1787827399.177484:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085371008:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787827399.177504:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827399.177524:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827399/real 1787827399] req@ffff9e35da7192c0 x1874672085371008/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827424 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827399.177546:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7192c0 x1874672085371008/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827424 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827399.177555:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827399.177558:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827399.177560:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085371008:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787827399.177565:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085371136:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787827399.177581:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827399.177589:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827399/real 1787827399] req@ffff9e35da718000 x1874672085371136/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827424 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827399.177596:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718000 x1874672085371136/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827424 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827399.177602:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827399.177604:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827399.177606:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085371136:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787827399.177609:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085371264:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787827399.177619:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827399.177625:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827399/real 1787827399] req@ffff9e35da71a1c0 x1874672085371264/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827424 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827399.177632:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a1c0 x1874672085371264/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827424 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827399.177641:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827399.177643:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827399.177645:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085371264:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787827399.177648:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085371392:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787827399.177658:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827399.177664:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827399/real 1787827399] req@ffff9e35da719a40 x1874672085371392/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827424 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827399.177676:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672085371392/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827424 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827399.177685:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827399.177688:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827399.177692:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085371392:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787827399.177698:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085371520:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787827399.177717:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827399.177729:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827399/real 1787827399] req@ffff9e35da718b40 x1874672085371520/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827424 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827399.177741:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718b40 x1874672085371520/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827424 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827399.177748:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827399.177750:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827399.177753:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085371520:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787827399.177757:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085371648:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787827399.177778:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827399.177785:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827399/real 1787827399] req@ffff9e35da718f00 x1874672085371648/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827424 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827399.177792:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672085371648/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827424 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827399.177817:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827399.177819:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827399.177821:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085371648:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787827399.177825:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085371776:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787827399.177838:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827399.177845:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827399/real 1787827399] req@ffff9e35da71a940 x1874672085371776/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827424 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827399.177852:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672085371776/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827424 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827399.177858:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827399.177860:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827399.177862:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085371776:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787827399.177865:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085371904:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787827399.177874:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827399.177881:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827399/real 1787827399] req@ffff9e35da7183c0 x1874672085371904/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827424 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827399.177888:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672085371904/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827424 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827399.177893:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827399.177895:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827399.177896:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085371904:192.168.203.141@tcp:8:kworker.0 00010000:00010000:3.0:1787827399.303920:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827401.351988:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:3.0:1787827402.375975:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787827402.376076:0:15664:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827403.399940:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787827404.296249:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827404.296265:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787827404.296271:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827404.296288:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 839 00000100:00080000:3.0:1787827404.296305:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827404.296346:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787827404.296404:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da772580 x1874672085372032/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827404.296475:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827404.296483:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827404.296487:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827404.296495:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 839 00000100:00080000:3.0:1787827404.296503:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827404.296511:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827404.296529:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827404.296554:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7712c0 x1874672085372160/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827404.296574:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827404.296581:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787827404.296585:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827404.296592:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 839 00000100:00080000:3.0:1787827404.296598:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827404.296606:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827404.296614:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827404.296631:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da771a40 x1874672085372288/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827404.296648:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827404.296654:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787827404.296658:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827404.296666:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 839 00000100:00080000:3.0:1787827404.296672:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827404.296679:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827404.296688:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:3.0:1787827404.296708:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da770f00 x1874672085372416/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827404.296725:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827404.296733:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787827404.296736:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827404.296743:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 839 00000100:00080000:3.0:1787827404.296749:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827404.296756:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827404.296764:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:3.0:1787827404.296783:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da771e00 x1874672085372544/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827404.296895:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827404.296902:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827404.296905:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827404.296914:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 839 00000100:00080000:3.0:1787827404.296922:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827404.296929:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827404.296938:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827404.296961:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da772940 x1874672085372672/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827404.296979:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827404.296985:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787827404.296989:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827404.296996:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 839 00000100:00080000:3.0:1787827404.297002:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827404.297008:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827404.297018:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827404.297037:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da770000 x1874672085372800/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827404.297055:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827404.297061:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787827404.297065:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827404.297073:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 839 00000100:00080000:3.0:1787827404.297079:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827404.297087:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827404.297095:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:3.0:1787827404.297135:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7730c0 x1874672085372928/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827404.297161:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827404.297167:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787827404.297171:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827404.297178:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 839 00000100:00080000:3.0:1787827404.297185:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827404.297202:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827404.297211:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:3.0:1787827404.297234:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b840 x1874672085373056/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:3.0:1787827406.471932:0:15664:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827410.567998:0:15664:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787827419.655983:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827419.655997:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787827419.656003:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827419.656020:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 855 00000100:00080000:3.0:1787827419.656404:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827419.656419:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787827419.656506:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649bc00 x1874672085375488/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827419.656619:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827419.656629:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827419.656633:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827419.656648:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 855 00000100:00080000:3.0:1787827419.656656:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827419.656663:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827419.656680:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827419.656705:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a940 x1874672085375616/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827419.656723:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827419.656729:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787827419.656732:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827419.656739:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 855 00000100:00080000:3.0:1787827419.656746:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827419.656754:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827419.656766:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827419.656791:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498b40 x1874672085375744/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827419.657064:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827419.657071:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787827419.657075:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827419.657083:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 855 00000100:00080000:3.0:1787827419.657093:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827419.657107:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827419.657118:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:3.0:1787827419.657224:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498780 x1874672085375872/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827419.657246:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827419.657257:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787827419.657264:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827419.657276:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 855 00000100:00080000:3.0:1787827419.657286:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827419.657299:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827419.657309:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:3.0:1787827419.657335:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499680 x1874672085376000/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827419.657361:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827419.657368:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827419.657371:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827419.657379:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 855 00000100:00080000:3.0:1787827419.657387:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827419.657399:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827419.657408:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827419.657432:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a580 x1874672085376128/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827419.657458:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827419.657466:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787827419.657470:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827419.657484:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 855 00000100:00080000:3.0:1787827419.657493:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827419.657505:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827419.657515:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827419.657538:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498f00 x1874672085376256/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827419.657555:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827419.657561:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787827419.657564:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827419.657575:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 855 00000100:00080000:3.0:1787827419.657583:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827419.657592:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827419.657604:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:3.0:1787827419.657629:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649ad00 x1874672085376384/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827419.657659:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827419.657671:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787827419.657675:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827419.657686:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 855 00000100:00080000:3.0:1787827419.657696:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827419.657702:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827419.657710:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:3.0:1787827419.657733:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499a40 x1874672085376512/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787827421.832006:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00080000:3.0:1787827424.776050:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827424.776063:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787827424.776068:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827424.776084:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 860 00000100:00080000:3.0:1787827424.776102:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827424.776116:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787827424.776207:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498000 x1874672085376640/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827424.776264:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827424.776271:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827424.776275:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827424.776283:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 860 00000100:00080000:3.0:1787827424.776292:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827424.776302:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827424.776319:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827424.776351:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c64992c0 x1874672085376768/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827424.776376:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827424.776383:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787827424.776387:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827424.776394:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 860 00000100:00080000:3.0:1787827424.776403:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827424.776412:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827424.776422:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827424.776446:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a1c0 x1874672085376896/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827424.776478:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827424.776485:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787827424.776499:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827424.776510:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 860 00000100:00080000:3.0:1787827424.776517:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827424.776526:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827424.776537:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:3.0:1787827424.776566:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c64983c0 x1874672085377024/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827424.776589:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827424.776597:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787827424.776601:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827424.776610:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 860 00000100:00080000:3.0:1787827424.776621:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827424.776633:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827424.776642:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:3.0:1787827424.776666:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499e00 x1874672085377152/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827424.776702:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827424.776713:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827424.776723:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827424.776737:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 860 00000100:00080000:3.0:1787827424.776744:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827424.776752:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827424.776762:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827424.776791:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b480 x1874672085377280/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827424.776996:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827424.777004:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787827424.777009:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827424.777020:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 860 00000100:00080000:3.0:1787827424.777036:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827424.777049:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827424.777064:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827424.777098:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b0c0 x1874672085377408/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827424.777126:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827424.777142:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787827424.777152:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827424.777195:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 860 00000100:00080000:3.0:1787827424.777213:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827424.777235:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827424.777257:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:3.0:1787827424.777294:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498000 x1874672085377536/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827424.777316:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827424.777322:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787827424.777329:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827424.777339:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 860 00000100:00080000:3.0:1787827424.777348:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827424.777355:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827424.777365:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:3.0:1787827424.777386:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499a40 x1874672085377664/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787827426.952072:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787827427.975888:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787827428.999921:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827433.095916:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827437.191975:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827439.239975:0:15664:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827440.263974:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827441.288033:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827443.335923:0:15664:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827444.360024:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827445.384024:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827446.407953:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827447.431953:0:15664:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827448.456032:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827449.479910:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787827450.375891:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827450.375899:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787827450.375904:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827450.375914:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 885 00000100:00080000:3.0:1787827450.375937:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827450.375945:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787827450.375984:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649ad00 x1874672085382400/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827450.376146:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827450.376152:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827450.376154:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827450.376161:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 885 00000100:00080000:3.0:1787827450.376166:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827450.376170:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827450.376181:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827450.376200:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498f00 x1874672085382528/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827450.376227:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827450.376231:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787827450.376233:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827450.376238:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 885 00000100:00080000:3.0:1787827450.376243:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827450.376248:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827450.376254:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827450.376269:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a580 x1874672085382656/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827450.376280:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827450.376283:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787827450.376285:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827450.376292:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 885 00000100:00080000:3.0:1787827450.376299:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827450.376304:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827450.376310:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:3.0:1787827450.376323:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499680 x1874672085382784/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827450.376333:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827450.376337:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787827450.376339:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827450.376347:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 885 00000100:00080000:3.0:1787827450.376354:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827450.376361:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827450.376367:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:3.0:1787827450.376387:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498780 x1874672085382912/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827450.376420:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827450.376425:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827450.376431:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827450.376438:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 885 00000100:00080000:3.0:1787827450.376442:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827450.376448:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827450.376457:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827450.376475:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498b40 x1874672085383040/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827450.376487:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827450.376495:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787827450.376498:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827450.376504:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 885 00000100:00080000:3.0:1787827450.376508:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827450.376512:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827450.376517:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827450.376533:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a940 x1874672085383168/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827450.376548:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827450.376557:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787827450.376562:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827450.376570:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 885 00000100:00080000:3.0:1787827450.376577:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827450.376586:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827450.376594:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:3.0:1787827450.376611:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649bc00 x1874672085383296/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827450.376631:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827450.376638:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787827450.376642:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827450.376647:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 885 00000100:00080000:3.0:1787827450.376652:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827450.376657:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827450.376662:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:3.0:1787827450.376678:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b840 x1874672085383424/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:3.0:1787827451.527934:0:15664:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827453.575903:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827455.623921:0:15664:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827456.647921:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827457.671937:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827459.720024:0:15664:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827460.743935:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827461.768013:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827463.815958:0:15664:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827464.839981:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827465.863976:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827467.911946:0:15664:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827468.935934:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827469.959942:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827470.983972:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827472.007959:0:15664:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827473.031919:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827475.079923:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827477.127931:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827479.175927:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827483.271949:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787827506.695925:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827506.695937:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787827506.695941:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827506.695951:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 942 00000100:00080000:3.0:1787827506.695961:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827506.695967:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787827506.696001:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b840 x1874672085395072/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827506.696027:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827506.696030:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827506.696031:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827506.696034:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 942 00000100:00080000:3.0:1787827506.696037:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827506.696040:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827506.696054:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827506.696064:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649bc00 x1874672085395200/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827506.696095:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827506.696106:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787827506.696109:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827506.696114:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 942 00000100:00080000:3.0:1787827506.696118:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827506.696122:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827506.696126:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827506.696139:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a940 x1874672085395328/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827506.696170:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827506.696173:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787827506.696175:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827506.696179:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 942 00000100:00080000:3.0:1787827506.696183:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827506.696185:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827506.696189:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:3.0:1787827506.696204:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498b40 x1874672085395456/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827506.696214:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827506.696233:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787827506.696235:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827506.696239:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 942 00000100:00080000:3.0:1787827506.696243:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827506.696249:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827506.696257:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:3.0:1787827506.696279:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498780 x1874672085395584/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827506.696340:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827506.696344:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827506.696346:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827506.696353:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 942 00000100:00080000:3.0:1787827506.696360:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827506.696368:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827506.696376:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827506.696406:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499680 x1874672085395712/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827506.696419:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827506.696422:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787827506.696423:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827506.696426:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 942 00000100:00080000:3.0:1787827506.696428:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827506.696430:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827506.696434:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827506.696451:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a580 x1874672085395840/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827506.696483:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827506.696489:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787827506.696494:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827506.696499:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 942 00000100:00080000:3.0:1787827506.696504:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827506.696509:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827506.696518:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:3.0:1787827506.696553:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498f00 x1874672085395968/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827506.696572:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827506.696578:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787827506.696582:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827506.696591:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 942 00000100:00080000:3.0:1787827506.696596:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827506.696621:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827506.696628:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:3.0:1787827506.696660:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649ad00 x1874672085396096/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827511.815892:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827511.815903:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787827511.815909:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827511.815920:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 947 00000100:00080000:3.0:1787827511.815932:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827511.815940:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787827511.815983:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499a40 x1874672085396224/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827511.816011:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827511.816014:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827511.816015:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827511.816019:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 947 00000100:00080000:3.0:1787827511.816021:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827511.816024:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827511.816034:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827511.816045:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498000 x1874672085396352/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827511.816052:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827511.816055:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787827511.816056:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827511.816059:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 947 00000100:00080000:3.0:1787827511.816061:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827511.816063:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827511.816066:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827511.816076:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b0c0 x1874672085396480/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827511.816103:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827511.816106:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787827511.816107:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827511.816111:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 947 00000100:00080000:3.0:1787827511.816114:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827511.816116:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827511.816120:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:3.0:1787827511.816129:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b480 x1874672085396608/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827511.816136:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827511.816139:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787827511.816140:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827511.816143:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 947 00000100:00080000:3.0:1787827511.816145:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827511.816148:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827511.816151:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:3.0:1787827511.816162:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499e00 x1874672085396736/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827511.816171:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827511.816173:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827511.816174:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827511.816177:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 947 00000100:00080000:3.0:1787827511.816180:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827511.816183:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827511.816186:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827511.816196:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c64983c0 x1874672085396864/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827511.816213:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827511.816221:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787827511.816224:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827511.816232:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 947 00000100:00080000:3.0:1787827511.816235:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827511.816238:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827511.816244:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827511.816263:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a1c0 x1874672085396992/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827511.816283:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827511.816289:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787827511.816291:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827511.816297:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 947 00000100:00080000:3.0:1787827511.816303:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827511.816310:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827511.816316:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:3.0:1787827511.816331:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c64992c0 x1874672085397120/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827511.816363:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827511.816370:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787827511.816374:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827511.816380:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 947 00000100:00080000:3.0:1787827511.816385:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827511.816401:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827511.816409:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:3.0:1787827511.816427:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498000 x1874672085397248/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:3.0:1787827512.967952:0:33:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 11/11 locks to regular blp list, 1 blwis in pool 00000100:00080000:3.0:1787827516.935882:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827516.935892:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787827516.935896:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827516.935908:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 952 00000100:00080000:3.0:1787827516.935919:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827516.935926:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787827516.935972:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499a40 x1874672085397632/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827516.936001:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827516.936009:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827516.936011:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827516.936016:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 952 00000100:00080000:3.0:1787827516.936020:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827516.936026:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827516.936036:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827516.936053:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649ad00 x1874672085397760/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827516.936062:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827516.936076:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787827516.936078:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827516.936080:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 952 00000100:00080000:3.0:1787827516.936083:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827516.936086:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827516.936090:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827516.936098:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498f00 x1874672085397888/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827516.936105:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827516.936107:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787827516.936108:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827516.936111:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 952 00000100:00080000:3.0:1787827516.936114:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827516.936116:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827516.936119:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:3.0:1787827516.936127:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a580 x1874672085398016/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827516.936135:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827516.936138:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787827516.936139:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827516.936143:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 952 00000100:00080000:3.0:1787827516.936145:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827516.936147:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827516.936150:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:3.0:1787827516.936159:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499680 x1874672085398144/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827516.936168:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827516.936177:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827516.936180:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827516.936187:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 952 00000100:00080000:3.0:1787827516.936190:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827516.936192:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827516.936196:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827516.936204:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498780 x1874672085398272/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827516.936226:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827516.936234:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787827516.936235:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827516.936239:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 952 00000100:00080000:3.0:1787827516.936242:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827516.936247:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827516.936255:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827516.936264:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498b40 x1874672085398400/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827516.936271:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827516.936274:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787827516.936275:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827516.936278:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 952 00000100:00080000:3.0:1787827516.936289:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827516.936297:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827516.936301:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:3.0:1787827516.936311:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a940 x1874672085398528/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827516.936324:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827516.936328:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787827516.936329:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827516.936333:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 952 00000100:00080000:3.0:1787827516.936336:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827516.936341:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827516.936344:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:3.0:1787827516.936364:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649bc00 x1874672085398656/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787827517.063947:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:3.0:1787827519.111935:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:3.0:1787827520.135915:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787827523.208010:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827524.231999:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1787827525.128041:0:2398:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.141:988 00000400:00000100:3.0:1787827525.128090:0:2398:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.141@tcp at host 192.168.203.141 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1787827525.128103:0:2398:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1787827525.128139:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000800:00000100:3.0:1787827525.128149:0:2398:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1787827525.128156:0:2398:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1787827525.128164:0:2398:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1787827525.128170:0:2398:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1787827525.128176:0:2398:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1787827525.128191:0:2398:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1787827525.128205:0:2398:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00010000:00010000:3.0:1787827526.279963:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827527.304026:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827529.351961:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827530.375988:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827532.424044:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827535.495971:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827536.519951:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:3.0:1787827537.543981:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787827538.568016:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827539.591973:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787827540.615906:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787827541.639958:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787827541.640047:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827543.688844:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787827544.711934:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787827544.712018:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827545.735928:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827545.735954:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787827546.759878:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787827547.783963:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827560.071921:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827561.095948:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827562.119929:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827563.016369:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085408000:192.168.203.141@tcp:250:kworker.0 00000800:00000100:3.0:1787827563.016691:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827563.016754:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827563/real 1787827563] req@ffff9e35da6f4b40 x1874672085408000/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827588 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827563.016774:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4b40 x1874672085408000/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827588 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827563.016786:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827563.016790:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827563.016794:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085408000:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1787827563.016867:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085408128:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787827563.016903:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827563.016924:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827563/real 1787827563] req@ffff9e35da6f43c0 x1874672085408128/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827588 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827563.016958:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f43c0 x1874672085408128/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827588 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827563.016970:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827563.016973:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827563.016978:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085408128:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787827563.016985:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085408256:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787827563.017008:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827563.017020:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827563/real 1787827563] req@ffff9e35da6f4780 x1874672085408256/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827588 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827563.017031:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4780 x1874672085408256/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827588 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827563.017048:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827563.017052:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827563.017057:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085408256:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787827563.017066:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085408384:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787827563.017094:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827563.017111:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827563/real 1787827563] req@ffff9e35da6f7840 x1874672085408384/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827588 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827563.017137:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7840 x1874672085408384/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827588 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827563.017146:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827563.017149:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827563.017152:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085408384:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787827563.017159:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085408512:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787827563.017179:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827563.017196:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827563/real 1787827563] req@ffff9e35da6f61c0 x1874672085408512/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827588 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827563.017206:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f61c0 x1874672085408512/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827588 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827563.017234:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827563.017238:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827563.017241:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085408512:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787827563.017247:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085408640:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787827563.017267:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827563.017278:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827563/real 1787827563] req@ffff9e35da6f5a40 x1874672085408640/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827588 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827563.017289:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5a40 x1874672085408640/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827588 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827563.017296:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827563.017299:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827563.017302:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085408640:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787827563.017306:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085408768:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787827563.017319:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827563.017327:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827563/real 1787827563] req@ffff9e35da6f70c0 x1874672085408768/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827588 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827563.017337:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f70c0 x1874672085408768/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827588 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827563.017345:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827563.017347:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827563.017350:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085408768:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787827563.017354:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085408896:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787827563.017368:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827563.017375:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827563/real 1787827563] req@ffff9e35da6f5680 x1874672085408896/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827588 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827563.017385:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5680 x1874672085408896/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827588 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827563.017393:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827563.017395:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827563.017398:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085408896:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787827563.017403:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085409024:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787827563.017416:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827563.017424:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827563/real 1787827563] req@ffff9e35da6f6940 x1874672085409024/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827588 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827563.017434:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6940 x1874672085409024/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827588 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827563.017441:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827563.017443:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827563.017447:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085409024:192.168.203.141@tcp:8:kworker.0 00010000:00010000:3.0:1787827563.144094:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827565.192062:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827568.136219:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085409152:192.168.203.141@tcp:250:kworker.0 00000800:00000100:3.0:1787827568.136299:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827568.136353:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827568/real 1787827568] req@ffff9e35da71a580 x1874672085409152/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827593 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827568.136383:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672085409152/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827593 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827568.136411:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827568.136421:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827568.136430:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085409152:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1787827568.136445:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085409280:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787827568.136487:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827568.136502:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827568/real 1787827568] req@ffff9e35da7192c0 x1874672085409280/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827593 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827568.136522:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7192c0 x1874672085409280/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827593 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827568.136539:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827568.136547:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827568.136553:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085409280:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787827568.136589:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085409408:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787827568.136627:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827568.136643:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827568/real 1787827568] req@ffff9e35da718000 x1874672085409408/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827593 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827568.136659:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718000 x1874672085409408/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827593 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827568.136666:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827568.136668:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827568.136671:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085409408:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787827568.136676:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085409536:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787827568.136693:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827568.136701:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827568/real 1787827568] req@ffff9e35da71a1c0 x1874672085409536/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827593 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827568.136709:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a1c0 x1874672085409536/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827593 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827568.136716:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827568.136718:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827568.136720:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085409536:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787827568.136724:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085409664:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787827568.136736:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827568.136742:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827568/real 1787827568] req@ffff9e35da719a40 x1874672085409664/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827593 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827568.136756:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672085409664/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827593 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827568.136762:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827568.136764:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827568.136766:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085409664:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787827568.136770:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085409792:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787827568.136782:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827568.136789:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827568/real 1787827568] req@ffff9e35da718b40 x1874672085409792/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827593 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827568.136798:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718b40 x1874672085409792/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827593 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827568.136838:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827568.136841:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827568.136843:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085409792:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787827568.136848:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085409920:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787827568.136863:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827568.136871:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827568/real 1787827568] req@ffff9e35da718f00 x1874672085409920/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827593 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827568.136879:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672085409920/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827593 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827568.136891:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827568.136893:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827568.136896:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085409920:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787827568.136900:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085410048:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787827568.136912:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827568.136918:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827568/real 1787827568] req@ffff9e35da71a940 x1874672085410048/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827593 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827568.136926:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672085410048/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827593 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827568.136933:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827568.136934:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827568.136936:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085410048:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787827568.136953:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085410176:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787827568.136964:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827568.136971:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827568/real 1787827568] req@ffff9e35da7183c0 x1874672085410176/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827593 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827568.136980:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672085410176/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827593 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827568.136986:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827568.136988:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827568.136991:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085410176:192.168.203.141@tcp:8:kworker.0 00010000:00010000:3.0:1787827568.263956:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827568.263996:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:3.0:1787827569.287988:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:3.0:1787827570.311948:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787827571.335971:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787827571.336055:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787827573.255948:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827573.255964:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787827573.255971:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827573.255985:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1008 00000100:00080000:3.0:1787827573.256000:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827573.256008:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787827573.256058:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b840 x1874672085410304/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827573.256088:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827573.256092:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827573.256095:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827573.256100:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1008 00000100:00080000:3.0:1787827573.256104:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827573.256109:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827573.256122:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827573.256153:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649bc00 x1874672085410432/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827573.256164:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827573.256168:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787827573.256170:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827573.256174:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1008 00000100:00080000:3.0:1787827573.256177:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827573.256181:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827573.256185:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827573.256196:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a940 x1874672085410560/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827573.256204:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827573.256208:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787827573.256209:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827573.256214:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1008 00000100:00080000:3.0:1787827573.256217:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827573.256221:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827573.256226:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:3.0:1787827573.256236:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498b40 x1874672085410688/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827573.256244:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827573.256248:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787827573.256249:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827573.256254:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1008 00000100:00080000:3.0:1787827573.256257:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827573.256261:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827573.256266:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:3.0:1787827573.256276:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498780 x1874672085410816/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827573.256286:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827573.256290:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827573.256292:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827573.256298:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1008 00000100:00080000:3.0:1787827573.256301:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827573.256307:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827573.256313:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827573.256328:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499680 x1874672085410944/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827573.256338:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827573.256342:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787827573.256343:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827573.256347:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1008 00000100:00080000:3.0:1787827573.256350:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827573.256354:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827573.256359:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827573.256368:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a580 x1874672085411072/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827573.256377:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827573.256381:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787827573.256383:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827573.256388:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1008 00000100:00080000:3.0:1787827573.256391:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827573.256396:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827573.256401:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:3.0:1787827573.256411:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498f00 x1874672085411200/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827573.256420:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827573.256425:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787827573.256426:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827573.256431:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1008 00000100:00080000:3.0:1787827573.256434:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827573.256438:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827573.256442:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:3.0:1787827573.256452:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649ad00 x1874672085411328/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787827574.407984:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787827574.408061:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827575.431933:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787827576.456015:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787827577.479904:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787827577.479994:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827578.376069:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085411456:192.168.203.141@tcp:250:kworker.0 00000800:00000100:3.0:1787827578.376158:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827578.376202:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827578/real 1787827578] req@ffff9e35c498ed00 x1874672085411456/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827603 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827578.376226:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498ed00 x1874672085411456/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827603 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827578.376259:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827578.376268:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827578.376275:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085411456:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1787827578.376306:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085411584:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787827578.376332:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827578.376347:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827578/real 1787827578] req@ffff9e35c498e1c0 x1874672085411584/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827603 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827578.376364:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e1c0 x1874672085411584/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827603 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827578.376393:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827578.376400:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827578.376406:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085411584:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787827578.376415:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085411712:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787827578.376436:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827578.376447:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827578/real 1787827578] req@ffff9e35c498c780 x1874672085411712/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827603 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827578.376471:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c780 x1874672085411712/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827603 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827578.376497:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827578.376502:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827578.376510:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085411712:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787827578.376518:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085411840:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787827578.376535:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827578.376542:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827578/real 1787827578] req@ffff9e35c498cf00 x1874672085411840/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827603 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827578.376550:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498cf00 x1874672085411840/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827603 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827578.376557:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827578.376559:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827578.376561:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085411840:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787827578.376564:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085411968:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787827578.376574:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827578.376579:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827578/real 1787827578] req@ffff9e35c498fc00 x1874672085411968/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827603 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827578.376586:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498fc00 x1874672085411968/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827603 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827578.376592:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827578.376594:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827578.376596:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085411968:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787827578.376609:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085412096:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787827578.376619:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827578.376625:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827578/real 1787827578] req@ffff9e35c498de00 x1874672085412096/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827603 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827578.376632:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498de00 x1874672085412096/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827603 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827578.376637:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827578.376639:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827578.376641:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085412096:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787827578.376644:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085412224:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787827578.376654:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827578.376659:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827578/real 1787827578] req@ffff9e35c498cb40 x1874672085412224/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827603 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827578.376667:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498cb40 x1874672085412224/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827603 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827578.376672:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827578.376673:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827578.376675:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085412224:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787827578.376679:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085412352:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787827578.376688:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827578.376694:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827578/real 1787827578] req@ffff9e35c498ed00 x1874672085412352/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827603 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827578.376701:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498ed00 x1874672085412352/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827603 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827578.376707:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827578.376708:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827578.376710:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085412352:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787827578.376713:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085412480:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787827578.376723:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827578.376728:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827578/real 1787827578] req@ffff9e35c498c3c0 x1874672085412480/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827603 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827578.376736:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c3c0 x1874672085412480/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827603 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827578.376741:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827578.376743:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827578.376745:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085412480:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787827578.503959:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787827580.552015:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827583.496029:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085412608:192.168.203.141@tcp:250:kworker.0 00000800:00000100:3.0:1787827583.496107:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827583.496150:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827583/real 1787827583] req@ffff9e35da6f5e00 x1874672085412608/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827608 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827583.496183:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5e00 x1874672085412608/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827608 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827583.496198:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827583.496203:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827583.496209:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085412608:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1787827583.496240:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085412736:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787827583.496274:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827583.496289:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827583/real 1787827583] req@ffff9e35da6f6580 x1874672085412736/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827608 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827583.496313:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6580 x1874672085412736/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827608 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827583.496328:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827583.496334:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827583.496341:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085412736:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787827583.496353:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085412864:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787827583.496379:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827583.496391:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827583/real 1787827583] req@ffff9e35da6f7480 x1874672085412864/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827608 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827583.496429:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7480 x1874672085412864/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827608 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827583.496440:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827583.496443:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827583.496445:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085412864:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787827583.496449:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085412992:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787827583.496462:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827583.496469:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827583/real 1787827583] req@ffff9e35da6f4000 x1874672085412992/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827608 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827583.496476:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4000 x1874672085412992/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827608 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827583.496481:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827583.496483:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827583.496485:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085412992:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787827583.496489:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085413120:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787827583.496499:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827583.496504:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827583/real 1787827583] req@ffff9e35da6f6d00 x1874672085413120/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827608 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827583.496518:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6d00 x1874672085413120/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827608 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827583.496524:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827583.496526:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827583.496529:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085413120:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787827583.496533:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085413248:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787827583.496544:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827583.496551:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827583/real 1787827583] req@ffff9e35da6f4f00 x1874672085413248/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827608 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827583.496558:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4f00 x1874672085413248/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827608 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827583.496563:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827583.496565:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827583.496567:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085413248:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787827583.496580:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085413376:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787827583.496591:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827583.496597:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827583/real 1787827583] req@ffff9e35da6f5e00 x1874672085413376/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827608 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827583.496604:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5e00 x1874672085413376/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827608 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827583.496609:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827583.496611:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827583.496613:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085413376:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787827583.496616:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085413504:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787827583.496625:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827583.496631:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827583/real 1787827583] req@ffff9e35da6f6940 x1874672085413504/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827608 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827583.496639:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6940 x1874672085413504/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827608 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827583.496644:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827583.496645:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827583.496647:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085413504:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787827583.496650:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085413632:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787827583.496660:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787827583.496665:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827583/real 1787827583] req@ffff9e35da6f5680 x1874672085413632/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827608 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827583.496672:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5680 x1874672085413632/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827608 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827583.496677:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827583.496679:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787827583.496681:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085413632:192.168.203.141@tcp:8:kworker.0 00010000:00010000:3.0:1787827583.624012:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827585.671945:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827586.696063:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787827588.615935:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827588.615945:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787827588.615950:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827588.615961:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1023 00000100:00080000:3.0:1787827588.615974:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827588.615982:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787827588.616018:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499a40 x1874672085413760/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827588.616061:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827588.616067:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827588.616069:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827588.616073:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1023 00000100:00080000:3.0:1787827588.616078:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827588.616081:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827588.616091:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827588.616109:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498000 x1874672085413888/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827588.616126:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827588.616129:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787827588.616132:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827588.616136:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1023 00000100:00080000:3.0:1787827588.616140:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827588.616144:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827588.616150:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827588.616163:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c64992c0 x1874672085414016/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827588.616174:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827588.616178:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787827588.616180:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827588.616184:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1023 00000100:00080000:3.0:1787827588.616188:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827588.616193:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827588.616200:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:3.0:1787827588.616216:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a1c0 x1874672085414144/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827588.616231:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827588.616235:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787827588.616237:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827588.616245:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1023 00000100:00080000:3.0:1787827588.616263:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827588.616272:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827588.616279:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:3.0:1787827588.616295:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c64983c0 x1874672085414272/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827588.616314:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827588.616318:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827588.616320:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827588.616325:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1023 00000100:00080000:3.0:1787827588.616329:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827588.616334:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827588.616342:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827588.616357:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499e00 x1874672085414400/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827588.616370:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827588.616374:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787827588.616376:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827588.616381:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1023 00000100:00080000:3.0:1787827588.616387:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827588.616396:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827588.616402:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827588.616415:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b480 x1874672085414528/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827588.616428:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827588.616433:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787827588.616435:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827588.616441:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1023 00000100:00080000:3.0:1787827588.616445:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827588.616453:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827588.616461:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:3.0:1787827588.616475:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b0c0 x1874672085414656/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827588.616488:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827588.616492:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787827588.616495:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827588.616504:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1023 00000100:00080000:3.0:1787827588.616511:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827588.616516:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827588.616522:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:3.0:1787827588.616537:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649ad00 x1874672085414784/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:3.0:1787827589.767925:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827590.792012:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1787827591.688071:0:2399:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.141:988 00000400:00000100:3.0:1787827591.688113:0:2399:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.141@tcp at host 192.168.203.141 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1787827591.688133:0:2399:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1787827591.688169:0:2399:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000800:00000100:3.0:1787827591.688187:0:2399:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1787827591.688195:0:2399:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1787827591.688208:0:2399:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1787827591.688225:0:2399:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1787827591.688243:0:2399:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1787827591.688258:0:2399:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1787827591.688276:0:2399:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:3.0:1787827591.815895:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787827593.863906:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827594.887922:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827595.911950:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:3.0:1787827596.935916:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787827596.935994:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827597.959957:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787827597.960057:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827598.983943:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:3.0:1787827600.007902:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787827600.007986:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827601.032072:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827603.079993:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787827603.976009:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827603.976026:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787827603.976032:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827603.976049:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1039 00000100:00080000:3.0:1787827603.976067:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827603.976079:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787827603.976157:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498f00 x1874672085417216/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827603.976208:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827603.976213:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827603.976215:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827603.976220:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1039 00000100:00080000:3.0:1787827603.976225:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827603.976231:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827603.976257:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827603.976290:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a580 x1874672085417344/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827603.976305:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827603.976309:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787827603.976322:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827603.976327:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1039 00000100:00080000:3.0:1787827603.976331:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827603.976335:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827603.976342:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827603.976357:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499680 x1874672085417472/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827603.976368:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827603.976372:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787827603.976374:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827603.976378:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1039 00000100:00080000:3.0:1787827603.976382:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827603.976396:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827603.976404:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:3.0:1787827603.976418:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498780 x1874672085417600/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827603.976428:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827603.976433:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787827603.976436:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827603.976445:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1039 00000100:00080000:3.0:1787827603.976451:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827603.976464:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827603.976479:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:3.0:1787827603.976507:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498b40 x1874672085417728/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827603.976558:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827603.976569:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827603.976576:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827603.976588:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1039 00000100:00080000:3.0:1787827603.976601:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827603.976608:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827603.976615:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827603.976632:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a940 x1874672085417856/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827603.976656:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827603.976661:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787827603.976663:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827603.976669:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1039 00000100:00080000:3.0:1787827603.976674:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827603.976678:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827603.976688:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827603.976707:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649bc00 x1874672085417984/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827603.976728:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827603.976740:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787827603.976745:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827603.976757:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1039 00000100:00080000:3.0:1787827603.976764:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827603.976772:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827603.976778:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:3.0:1787827603.976830:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b840 x1874672085418112/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827603.976843:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827603.976850:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787827603.976855:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827603.976873:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1039 00000100:00080000:3.0:1787827603.976886:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827603.976897:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827603.976905:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:3.0:1787827603.976947:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498f00 x1874672085418240/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:3.0:1787827604.104054:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827607.175995:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827608.199964:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787827609.095863:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827609.095873:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787827609.095877:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827609.095886:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1044 00000100:00080000:3.0:1787827609.095913:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827609.095920:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787827609.095969:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499a40 x1874672085418368/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827609.096012:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827609.096016:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827609.096018:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827609.096023:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1044 00000100:00080000:3.0:1787827609.096026:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827609.096030:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827609.096041:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827609.096056:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c64992c0 x1874672085418496/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827609.096069:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827609.096076:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787827609.096083:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827609.096092:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1044 00000100:00080000:3.0:1787827609.096095:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827609.096099:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827609.096104:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827609.096117:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498000 x1874672085418624/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827609.096127:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827609.096131:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787827609.096133:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827609.096137:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1044 00000100:00080000:3.0:1787827609.096143:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827609.096147:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827609.096152:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:3.0:1787827609.096170:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a1c0 x1874672085418752/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827609.096181:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827609.096185:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787827609.096187:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827609.096190:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1044 00000100:00080000:3.0:1787827609.096195:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827609.096199:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827609.096204:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:3.0:1787827609.096220:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c64983c0 x1874672085418880/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827609.096242:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827609.096246:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827609.096248:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827609.096254:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1044 00000100:00080000:3.0:1787827609.096261:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827609.096270:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827609.096280:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827609.096294:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499e00 x1874672085419008/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827609.096334:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827609.096337:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787827609.096339:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827609.096344:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1044 00000100:00080000:3.0:1787827609.096347:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827609.096350:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827609.096354:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827609.096367:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b480 x1874672085419136/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827609.096378:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827609.096381:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787827609.096382:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827609.096386:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1044 00000100:00080000:3.0:1787827609.096389:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827609.096397:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827609.096403:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:3.0:1787827609.096416:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b0c0 x1874672085419264/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827609.096458:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827609.096461:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787827609.096463:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827609.096470:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1044 00000100:00080000:3.0:1787827609.096477:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827609.096484:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827609.096492:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:3.0:1787827609.096505:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649ad00 x1874672085419392/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787827610.247971:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787827610.248109:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827611.272182:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787827614.215881:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827614.215909:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787827614.215916:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827614.215932:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1049 00000100:00080000:3.0:1787827614.215947:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827614.215975:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787827614.216055:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649ad00 x1874672085419520/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827614.216086:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827614.216091:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827614.216093:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827614.216098:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1049 00000100:00080000:3.0:1787827614.216102:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827614.216107:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827614.216119:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827614.216137:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b0c0 x1874672085419648/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827614.216153:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827614.216158:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787827614.216160:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827614.216172:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1049 00000100:00080000:3.0:1787827614.216178:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827614.216194:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827614.216201:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827614.216218:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b480 x1874672085419776/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827614.216228:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827614.216231:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787827614.216232:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827614.216236:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1049 00000100:00080000:3.0:1787827614.216239:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827614.216242:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827614.216245:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:3.0:1787827614.216264:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499e00 x1874672085419904/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827614.216272:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827614.216275:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787827614.216276:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827614.216279:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1049 00000100:00080000:3.0:1787827614.216282:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827614.216285:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827614.216288:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:3.0:1787827614.216295:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c64983c0 x1874672085420032/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827614.216306:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827614.216310:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827614.216318:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827614.216328:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1049 00000100:00080000:3.0:1787827614.216335:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827614.216343:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827614.216350:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827614.216393:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a1c0 x1874672085420160/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827614.216423:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827614.216447:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787827614.216456:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827614.216467:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1049 00000100:00080000:3.0:1787827614.216474:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827614.216489:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827614.216513:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827614.216532:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498000 x1874672085420288/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827614.216602:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827614.216606:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787827614.216610:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827614.216616:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1049 00000100:00080000:3.0:1787827614.216619:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827614.216625:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827614.216634:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:3.0:1787827614.216656:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c64992c0 x1874672085420416/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827614.216698:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827614.216703:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787827614.216708:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827614.216719:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1049 00000100:00080000:3.0:1787827614.216724:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827614.216752:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827614.216762:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:3.0:1787827614.216785:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499a40 x1874672085420544/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:3.0:1787827614.344018:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827616.391968:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787827617.415923:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787827617.416013:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827618.440036:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827619.464048:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827620.488009:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787827620.488049:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827621.511900:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787827621.511981:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827622.535910:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787827622.535972:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827623.559952:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787827623.560065:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827624.584014:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827625.608019:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787827625.608126:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827626.632030:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827627.656059:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827628.679939:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827629.703904:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827630.728016:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827631.752006:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827632.775926:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787827632.776003:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827633.800036:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827634.824078:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827635.847901:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787827635.847962:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827636.871894:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787827636.871956:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827637.895909:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787827637.895987:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827638.919924:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827639.943894:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787827639.943950:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827640.967915:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787827640.967987:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827641.991932:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787827641.992004:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827643.015975:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827644.039962:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827645.063949:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827647.111978:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827650.183911:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827652.231923:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827655.303943:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827656.327990:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827658.375970:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827659.399954:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827660.423948:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827661.448019:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827662.471973:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827663.496011:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827664.519989:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827665.543912:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827666.567956:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827667.591941:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827668.615968:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827669.639937:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827676.807927:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827676.807959:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787827677.832004:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787827679.879938:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827682.951943:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827683.975905:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827685.000029:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:3.0:1787827686.023883:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787827686.023955:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827687.048034:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827689.095924:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827690.119990:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827692.167919:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827694.216006:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827696.263920:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827697.287919:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827699.335916:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827700.359960:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827701.383982:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827703.432049:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827704.455974:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827706.503963:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827707.528006:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827708.551992:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:3.0:1787827709.575871:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787827709.575941:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827710.600014:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827711.623966:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787827713.671975:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827716.744003:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827717.767959:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827718.791970:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827719.815970:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827721.864049:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827722.888027:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827724.935967:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827725.959893:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787827725.959994:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787827726.855932:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827726.855940:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787827726.855944:0:8:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827726.855952:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1162 00000100:00080000:3.0:1787827726.855960:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827726.855965:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787827726.855994:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498f00 x1874672085444864/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827726.856019:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827726.856022:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827726.856023:0:8:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827726.856026:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1162 00000100:00080000:3.0:1787827726.856029:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827726.856031:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827726.856039:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827726.856048:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b840 x1874672085444992/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827726.856056:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827726.856059:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787827726.856060:0:8:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827726.856065:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1162 00000100:00080000:3.0:1787827726.856085:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827726.856089:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827726.856093:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827726.856104:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649bc00 x1874672085445120/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827726.856127:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827726.856129:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787827726.856131:0:8:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827726.856134:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1162 00000100:00080000:3.0:1787827726.856144:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827726.856146:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827726.856149:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:3.0:1787827726.856158:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a940 x1874672085445248/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827726.856166:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827726.856170:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787827726.856172:0:8:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827726.856176:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1162 00000100:00080000:3.0:1787827726.856180:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827726.856188:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827726.856206:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:3.0:1787827726.856220:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498b40 x1874672085445376/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827726.856231:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827726.856235:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827726.856237:0:8:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827726.856241:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1162 00000100:00080000:3.0:1787827726.856247:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827726.856255:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827726.856265:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827726.856286:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498780 x1874672085445504/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827726.856324:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827726.856330:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787827726.856334:0:8:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827726.856342:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1162 00000100:00080000:3.0:1787827726.856349:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827726.856359:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827726.856364:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827726.856377:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499680 x1874672085445632/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827726.856394:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827726.856400:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787827726.856402:0:8:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827726.856408:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1162 00000100:00080000:3.0:1787827726.856416:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827726.856423:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827726.856432:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:3.0:1787827726.856451:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a580 x1874672085445760/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827726.856509:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827726.856516:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787827726.856521:0:8:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827726.856529:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1162 00000100:00080000:3.0:1787827726.856537:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827726.856544:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827726.856552:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:3.0:1787827726.856568:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498f00 x1874672085445888/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787827726.983889:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787827728.008110:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827729.032005:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827732.103974:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827735.176055:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827737.223955:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827738.248110:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827740.296051:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827741.319944:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827743.367938:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827743.367990:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:3.0:1787827744.391888:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787827744.391986:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827745.415954:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827746.439981:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827748.488070:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827749.512120:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827751.560149:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787827752.455939:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827752.455951:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787827752.455955:0:8:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827752.455968:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1187 00000100:00080000:3.0:1787827752.455981:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827752.455991:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787827752.456042:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499a40 x1874672085450624/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827752.456069:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827752.456072:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827752.456074:0:8:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827752.456077:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1187 00000100:00080000:3.0:1787827752.456080:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827752.456083:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827752.456094:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827752.456105:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c64992c0 x1874672085450752/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827752.456113:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827752.456116:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787827752.456117:0:8:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827752.456121:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1187 00000100:00080000:3.0:1787827752.456124:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827752.456127:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827752.456131:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827752.456139:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498000 x1874672085450880/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827752.456146:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827752.456149:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787827752.456150:0:8:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827752.456154:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1187 00000100:00080000:3.0:1787827752.456155:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827752.456161:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827752.456165:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:3.0:1787827752.456174:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a1c0 x1874672085451008/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827752.456183:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827752.456186:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787827752.456187:0:8:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827752.456191:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1187 00000100:00080000:3.0:1787827752.456193:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827752.456195:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827752.456200:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:3.0:1787827752.456209:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c64983c0 x1874672085451136/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827752.456215:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827752.456218:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827752.456219:0:8:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827752.456226:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1187 00000100:00080000:3.0:1787827752.456242:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827752.456247:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827752.456253:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827752.456275:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499e00 x1874672085451264/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827752.456288:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827752.456292:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787827752.456294:0:8:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827752.456301:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1187 00000100:00080000:3.0:1787827752.456307:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827752.456319:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827752.456336:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827752.456350:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b480 x1874672085451392/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827752.456368:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827752.456381:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787827752.456385:0:8:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827752.456401:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1187 00000100:00080000:3.0:1787827752.456406:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827752.456417:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827752.456424:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:3.0:1787827752.456437:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b0c0 x1874672085451520/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827752.456460:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827752.456468:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787827752.456472:0:8:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827752.456483:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1187 00000100:00080000:3.0:1787827752.456486:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827752.456489:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827752.456494:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:3.0:1787827752.456505:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649ad00 x1874672085451648/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:3.0:1787827752.584013:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827754.631951:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827755.656067:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827756.679973:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787827757.575913:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827757.575921:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787827757.575924:0:8:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827757.575932:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1192 00000100:00080000:3.0:1787827757.575941:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827757.575947:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787827757.575980:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649ad00 x1874672085451776/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827757.576004:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827757.576007:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827757.576009:0:8:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827757.576012:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1192 00000100:00080000:3.0:1787827757.576014:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827757.576017:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827757.576025:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827757.576034:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b0c0 x1874672085451904/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827757.576042:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827757.576045:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787827757.576049:0:8:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827757.576053:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1192 00000100:00080000:3.0:1787827757.576059:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827757.576063:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827757.576066:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827757.576074:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b480 x1874672085452032/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827757.576080:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827757.576082:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787827757.576083:0:8:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827757.576086:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1192 00000100:00080000:3.0:1787827757.576088:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827757.576091:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827757.576094:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:3.0:1787827757.576101:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499e00 x1874672085452160/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827757.576107:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827757.576110:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787827757.576112:0:8:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827757.576114:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1192 00000100:00080000:3.0:1787827757.576116:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827757.576118:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827757.576122:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:3.0:1787827757.576131:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c64983c0 x1874672085452288/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827757.576138:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827757.576141:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827757.576143:0:8:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827757.576146:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1192 00000100:00080000:3.0:1787827757.576148:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827757.576151:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827757.576153:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827757.576160:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a1c0 x1874672085452416/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827757.576165:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827757.576167:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787827757.576168:0:8:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827757.576175:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1192 00000100:00080000:3.0:1787827757.576177:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827757.576183:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827757.576189:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827757.576199:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498000 x1874672085452544/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827757.576226:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827757.576229:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787827757.576230:0:8:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827757.576233:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1192 00000100:00080000:3.0:1787827757.576236:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827757.576240:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827757.576247:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:3.0:1787827757.576257:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c64992c0 x1874672085452672/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827757.576272:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827757.576277:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787827757.576281:0:8:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827757.576286:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1192 00000100:00080000:3.0:1787827757.576292:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827757.576296:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827757.576299:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:3.0:1787827757.576305:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499a40 x1874672085452800/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:3.0:1787827757.704021:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827758.727952:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827759.751936:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827760.775913:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827762.824041:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827763.847921:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827764.871954:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827765.895989:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827766.919950:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827768.967943:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827769.991976:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827771.015943:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827772.039989:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827774.087933:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787827774.088009:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827775.112000:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827780.231952:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827780.232008:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00080000:3.0:1787827783.175887:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827783.175899:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787827783.175905:0:8:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827783.175916:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1218 00000100:00080000:3.0:1787827783.175928:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827783.175936:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787827783.175977:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498f00 x1874672085457536/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827783.176007:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827783.176016:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827783.176019:0:8:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827783.176026:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1218 00000100:00080000:3.0:1787827783.176031:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827783.176040:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827783.176056:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827783.176078:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a580 x1874672085457664/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827783.176093:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827783.176098:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787827783.176100:0:8:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827783.176106:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1218 00000100:00080000:3.0:1787827783.176112:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827783.176121:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827783.176130:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827783.176147:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499680 x1874672085457792/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827783.176159:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827783.176163:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787827783.176166:0:8:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827783.176173:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1218 00000100:00080000:3.0:1787827783.176177:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827783.176196:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827783.176206:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:3.0:1787827783.176224:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498780 x1874672085457920/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827783.176238:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827783.176244:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787827783.176267:0:8:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827783.176276:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1218 00000100:00080000:3.0:1787827783.176283:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827783.176288:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827783.176294:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:3.0:1787827783.176309:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498b40 x1874672085458048/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827783.176322:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827783.176327:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827783.176330:0:8:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827783.176341:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1218 00000100:00080000:3.0:1787827783.176349:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827783.176370:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827783.176377:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827783.176394:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a940 x1874672085458176/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827783.176408:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827783.176412:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787827783.176414:0:8:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827783.176418:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1218 00000100:00080000:3.0:1787827783.176421:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827783.176424:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827783.176429:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827783.176441:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649bc00 x1874672085458304/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827783.176452:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827783.176460:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787827783.176463:0:8:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827783.176472:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1218 00000100:00080000:3.0:1787827783.176476:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827783.176482:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827783.176487:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:3.0:1787827783.176500:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b840 x1874672085458432/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827783.176511:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827783.176516:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787827783.176520:0:8:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827783.176526:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1218 00000100:00080000:3.0:1787827783.176528:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827783.176531:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827783.176535:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:3.0:1787827783.176545:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498f00 x1874672085458560/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787827787.400020:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:3.0:1787827788.423965:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000400:00000100:3.0:1787827791.368048:0:2398:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.141:988 00000400:00000100:3.0:1787827791.368101:0:2398:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.141@tcp at host 192.168.203.141 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1787827791.368119:0:2398:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1787827791.368169:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000800:00000100:3.0:1787827791.368186:0:2398:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1787827791.368199:0:2398:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1787827791.368214:0:2398:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1787827791.368230:0:2398:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1787827791.368246:0:2398:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1787827791.368260:0:2398:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1787827791.368275:0:2398:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00080000:3.0:1787827793.416000:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827793.416008:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787827793.416012:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827793.416020:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1228 00000100:00080000:3.0:1787827793.416028:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827793.416033:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787827793.416063:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499a40 x1874672085459840/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827793.416091:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827793.416095:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827793.416097:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827793.416102:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1228 00000100:00080000:3.0:1787827793.416106:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827793.416111:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827793.416122:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827793.416137:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c64992c0 x1874672085459968/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827793.416177:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827793.416181:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787827793.416184:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827793.416189:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1228 00000100:00080000:3.0:1787827793.416194:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827793.416211:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827793.416218:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827793.416242:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498000 x1874672085460096/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827793.416256:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827793.416260:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787827793.416263:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827793.416267:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1228 00000100:00080000:3.0:1787827793.416272:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827793.416276:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827793.416282:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:3.0:1787827793.416303:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a1c0 x1874672085460224/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827793.416336:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827793.416340:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787827793.416342:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827793.416346:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1228 00000100:00080000:3.0:1787827793.416350:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827793.416355:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827793.416360:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:3.0:1787827793.416377:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c64983c0 x1874672085460352/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827793.416390:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827793.416400:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827793.416404:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827793.416410:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1228 00000100:00080000:3.0:1787827793.416414:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827793.416426:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827793.416433:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827793.416451:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499e00 x1874672085460480/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827793.416463:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827793.416466:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787827793.416468:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827793.416481:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1228 00000100:00080000:3.0:1787827793.416487:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827793.416493:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827793.416498:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827793.416511:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b480 x1874672085460608/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827793.416531:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827793.416541:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787827793.416545:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827793.416554:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1228 00000100:00080000:3.0:1787827793.416561:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827793.416569:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827793.416577:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:3.0:1787827793.416592:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b0c0 x1874672085460736/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827793.416626:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827793.416633:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787827793.416638:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827793.416644:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1228 00000100:00080000:3.0:1787827793.416652:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827793.416659:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827793.416665:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:3.0:1787827793.416682:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649ad00 x1874672085460864/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:3.0:1787827793.543986:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827795.591953:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787827796.616081:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827799.687919:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827802.760071:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827810.952007:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827814.024068:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827816.071884:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787827817.095898:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787827817.095972:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827818.119959:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787827819.144052:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827820.168074:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827821.192025:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787827822.215970:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787827822.216061:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827823.240115:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827825.287940:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827825.287970:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:3.0:1787827826.311919:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787827826.312027:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827828.359954:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827830.407940:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827831.432126:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827833.480004:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787827833.480109:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827834.503948:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827835.528077:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787827836.552028:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827837.575961:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827839.623949:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827840.648077:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827842.695969:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827843.720064:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827844.743990:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827847.815941:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827850.887983:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827852.935916:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827853.959998:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827854.856015:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827854.856020:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827854.856022:0:8:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827854.856026:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1290 00000100:00080000:2.0:1787827854.856029:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827854.856033:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787827854.856043:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827854.856058:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c780 x1874672085473792/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827854.856079:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827854.856083:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787827854.856086:0:8:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827854.856091:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1290 00000100:00080000:2.0:1787827854.856095:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827854.856099:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787827854.856104:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827854.856116:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cf00 x1874672085473920/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827854.856125:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827854.856129:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787827854.856130:0:8:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827854.856134:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1290 00000100:00080000:2.0:1787827854.856139:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827854.856144:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787827854.856150:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:2.0:1787827854.856162:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498fc00 x1874672085474048/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827854.856180:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827854.856199:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787827854.856201:0:8:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827854.856207:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1290 00000100:00080000:2.0:1787827854.856213:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827854.856218:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787827854.856227:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:2.0:1787827854.856246:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498de00 x1874672085474176/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827854.856267:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827854.856272:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827854.856275:0:8:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827854.856284:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1290 00000100:00080000:2.0:1787827854.856308:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827854.856315:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787827854.856325:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827854.856344:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498ed00 x1874672085474304/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827854.856370:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827854.856375:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787827854.856379:0:8:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827854.856402:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1290 00000100:00080000:2.0:1787827854.856407:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827854.856412:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787827854.856419:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827854.856437:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c3c0 x1874672085474432/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827854.856450:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827854.856471:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787827854.856474:0:8:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827854.856487:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1290 00000100:00080000:2.0:1787827854.856502:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827854.856512:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787827854.856522:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:2.0:1787827854.856547:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e940 x1874672085474560/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827854.856576:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827854.856614:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787827854.856616:0:8:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827854.856625:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1290 00000100:00080000:2.0:1787827854.856633:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827854.856637:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787827854.856644:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:2.0:1787827854.856660:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498d680 x1874672085474688/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787827854.983894:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787827856.007914:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787827856.007933:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787827856.008088:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787827857.031876:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787827857.031921:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1787827857.928090:0:2399:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.141:988 00000400:00000100:3.0:1787827857.928133:0:2399:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.141@tcp at host 192.168.203.141 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1787827857.928150:0:2399:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1787827857.928235:0:2399:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000800:00000100:3.0:1787827857.928254:0:2399:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1787827857.928268:0:2399:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1787827857.928281:0:2399:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1787827857.928301:0:2399:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1787827857.928320:0:2399:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1787827857.928341:0:2399:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1787827857.928359:0:2399:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:3.0:1787827858.055946:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787827859.080016:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787827859.080089:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787827859.976050:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085474816:192.168.203.141@tcp:250:kworker.0 00000800:00000100:2.0:1787827859.976128:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787827859.976196:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827859/real 1787827859] req@ffff9e35da71a580 x1874672085474816/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827884 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827859.976231:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672085474816/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827884 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827859.976256:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827859.976267:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787827859.976277:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085474816:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1787827859.976311:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085474944:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787827859.976347:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787827859.976364:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827859/real 1787827859] req@ffff9e35da7192c0 x1874672085474944/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827884 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827859.976379:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7192c0 x1874672085474944/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827884 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827859.976390:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827859.976394:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787827859.976398:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085474944:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787827859.976405:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085475072:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787827859.976437:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787827859.976450:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827859/real 1787827859] req@ffff9e35da718000 x1874672085475072/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827884 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827859.976464:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718000 x1874672085475072/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827884 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827859.976473:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827859.976477:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787827859.976480:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085475072:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787827859.976487:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085475200:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787827859.976508:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787827859.976519:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827859/real 1787827859] req@ffff9e35da71a1c0 x1874672085475200/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827884 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827859.976532:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a1c0 x1874672085475200/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827884 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827859.976542:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827859.976546:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787827859.976549:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085475200:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787827859.976555:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085475328:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787827859.976578:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787827859.976591:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827859/real 1787827859] req@ffff9e35da719a40 x1874672085475328/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827884 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827859.976606:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672085475328/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827884 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827859.976616:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827859.976620:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787827859.976624:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085475328:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787827859.976630:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085475456:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787827859.976653:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787827859.976665:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827859/real 1787827859] req@ffff9e35da718b40 x1874672085475456/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827884 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827859.976679:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718b40 x1874672085475456/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827884 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827859.976688:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827859.976692:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787827859.976696:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085475456:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787827859.976702:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085475584:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787827859.976722:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787827859.976734:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827859/real 1787827859] req@ffff9e35da718f00 x1874672085475584/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827884 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827859.976754:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672085475584/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827884 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827859.976762:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827859.976765:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787827859.976769:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085475584:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787827859.976775:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085475712:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787827859.976794:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787827859.976831:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827859/real 1787827859] req@ffff9e35da71a940 x1874672085475712/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827884 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827859.976845:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672085475712/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827884 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827859.976854:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827859.976858:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787827859.976862:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085475712:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787827859.976890:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085475840:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787827859.976911:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787827859.976924:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827859/real 1787827859] req@ffff9e35da7183c0 x1874672085475840/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827884 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827859.976936:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672085475840/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827884 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827859.976950:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827859.976953:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787827859.976957:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085475840:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787827860.103890:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787827860.103975:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787827861.127958:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787827862.151988:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787827862.152106:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827863.175932:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827864.199938:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787827864.199941:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787827865.096038:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085475968:192.168.203.141@tcp:250:kworker.0 00000800:00000100:2.0:1787827865.096121:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787827865.096167:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827865/real 1787827865] req@ffff9e35da6f5a40 x1874672085475968/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827890 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827865.096202:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5a40 x1874672085475968/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827890 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827865.096219:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827865.096227:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787827865.096235:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085475968:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1787827865.096268:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085476096:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787827865.096301:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787827865.096334:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827865/real 1787827865] req@ffff9e35da6f4b40 x1874672085476096/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827890 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827865.096365:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4b40 x1874672085476096/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827890 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827865.096375:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827865.096379:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787827865.096382:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085476096:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787827865.096389:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085476224:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787827865.096406:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787827865.096416:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827865/real 1787827865] req@ffff9e35da6f43c0 x1874672085476224/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827890 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827865.096428:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f43c0 x1874672085476224/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827890 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827865.096438:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827865.096441:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787827865.096444:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085476224:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787827865.096449:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085476352:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787827865.096465:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787827865.096474:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827865/real 1787827865] req@ffff9e35da6f4780 x1874672085476352/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827890 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827865.096485:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4780 x1874672085476352/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827890 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827865.096493:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827865.096495:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787827865.096499:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085476352:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787827865.096505:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085476480:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787827865.096518:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787827865.096527:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827865/real 1787827865] req@ffff9e35da6f7840 x1874672085476480/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827890 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827865.096538:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7840 x1874672085476480/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827890 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827865.096547:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827865.096550:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787827865.096554:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085476480:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787827865.096559:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085476608:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787827865.096578:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787827865.096588:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827865/real 1787827865] req@ffff9e35da6f61c0 x1874672085476608/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827890 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827865.096599:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f61c0 x1874672085476608/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827890 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827865.096609:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827865.096612:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787827865.096615:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085476608:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787827865.096620:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085476736:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787827865.096634:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787827865.096643:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827865/real 1787827865] req@ffff9e35da6f7480 x1874672085476736/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827890 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827865.096654:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7480 x1874672085476736/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827890 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827865.096662:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827865.096664:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787827865.096667:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085476736:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787827865.096688:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085476864:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787827865.096703:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787827865.096713:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827865/real 1787827865] req@ffff9e35da6f5a40 x1874672085476864/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827890 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827865.096725:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5a40 x1874672085476864/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827890 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827865.096734:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827865.096738:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787827865.096742:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085476864:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787827865.096747:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085476992:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787827865.096762:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787827865.096790:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827865/real 1787827865] req@ffff9e35da6f70c0 x1874672085476992/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827890 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827865.096820:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f70c0 x1874672085476992/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827890 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827865.096829:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827865.096844:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787827865.096847:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085476992:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787827865.223892:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787827865.223938:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787827866.247955:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787827866.247960:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787827867.271903:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787827867.271971:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787827868.295905:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787827868.295933:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827868.295952:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787827869.319914:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787827869.319921:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787827869.320030:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787827870.216346:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085477120:192.168.203.141@tcp:250:kworker.0 00000800:00000100:2.0:1787827870.216472:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787827870.216538:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827870/real 1787827870] req@ffff9e35da718780 x1874672085477120/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827895 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827870.216580:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718780 x1874672085477120/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827895 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827870.216619:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827870.216626:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787827870.216636:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085477120:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1787827870.216655:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085477248:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787827870.216692:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787827870.216711:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827870/real 1787827870] req@ffff9e35da7183c0 x1874672085477248/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827895 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827870.216763:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672085477248/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827895 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827870.216779:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827870.216791:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787827870.216827:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085477248:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787827870.216870:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085477376:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787827870.216902:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787827870.216918:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827870/real 1787827870] req@ffff9e35da71a940 x1874672085477376/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827895 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827870.216946:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672085477376/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827895 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827870.216957:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827870.216961:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787827870.216967:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085477376:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787827870.216976:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085477504:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787827870.217001:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787827870.217015:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827870/real 1787827870] req@ffff9e35da718f00 x1874672085477504/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827895 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827870.217035:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672085477504/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827895 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827870.217058:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827870.217063:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787827870.217068:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085477504:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787827870.217095:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085477632:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787827870.217144:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787827870.217165:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827870/real 1787827870] req@ffff9e35da718b40 x1874672085477632/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827895 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827870.217188:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718b40 x1874672085477632/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827895 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827870.217205:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827870.217211:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787827870.217217:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085477632:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787827870.217229:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085477760:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787827870.217264:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787827870.217285:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827870/real 1787827870] req@ffff9e35da719a40 x1874672085477760/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827895 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827870.217324:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672085477760/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827895 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827870.217341:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827870.217347:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787827870.217354:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085477760:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787827870.217365:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085477888:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787827870.217408:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787827870.217424:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827870/real 1787827870] req@ffff9e35da71a1c0 x1874672085477888/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827895 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827870.217443:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a1c0 x1874672085477888/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827895 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827870.217455:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827870.217460:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787827870.217465:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085477888:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787827870.217475:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085478016:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787827870.217501:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787827870.217517:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827870/real 1787827870] req@ffff9e35da718000 x1874672085478016/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827895 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827870.217536:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718000 x1874672085478016/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827895 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827870.217549:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827870.217555:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787827870.217560:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085478016:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787827870.217569:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085478144:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787827870.217595:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787827870.217610:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827870/real 1787827870] req@ffff9e35da7192c0 x1874672085478144/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827895 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827870.217629:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7192c0 x1874672085478144/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827895 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827870.217642:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827870.217647:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787827870.217652:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085478144:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787827870.344008:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787827871.367967:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787827871.368167:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787827872.391906:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787827872.391983:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787827873.415996:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:3.0:1787827874.440013:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787827874.440060:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787827875.336214:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085478272:192.168.203.141@tcp:250:kworker.0 00000800:00000100:2.0:1787827875.336351:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787827875.336435:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827875/real 1787827875] req@ffff9e35da71a580 x1874672085478272/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827900 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827875.336518:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672085478272/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827900 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827875.336566:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827875.336584:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787827875.336607:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085478272:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1787827875.336625:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085478400:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787827875.336671:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787827875.336688:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827875/real 1787827875] req@ffff9e35da7192c0 x1874672085478400/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827900 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827875.336743:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7192c0 x1874672085478400/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827900 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827875.336758:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827875.336776:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787827875.336788:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085478400:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787827875.336877:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085478528:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787827875.336919:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787827875.336944:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827875/real 1787827875] req@ffff9e35da718000 x1874672085478528/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827900 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827875.336958:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718000 x1874672085478528/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827900 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827875.336966:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827875.336969:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787827875.336972:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085478528:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787827875.336978:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085478656:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787827875.336998:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787827875.337007:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827875/real 1787827875] req@ffff9e35da71a1c0 x1874672085478656/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827900 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827875.337017:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a1c0 x1874672085478656/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827900 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827875.337024:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827875.337027:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787827875.337030:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085478656:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787827875.337058:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085478784:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787827875.337120:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787827875.337139:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827875/real 1787827875] req@ffff9e35da719a40 x1874672085478784/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827900 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827875.337159:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672085478784/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827900 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827875.337172:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827875.337175:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787827875.337179:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085478784:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787827875.337186:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085478912:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787827875.337226:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787827875.337237:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827875/real 1787827875] req@ffff9e35da718b40 x1874672085478912/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827900 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827875.337249:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718b40 x1874672085478912/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827900 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827875.337257:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827875.337260:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787827875.337264:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085478912:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787827875.337270:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085479040:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787827875.337287:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787827875.337295:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827875/real 1787827875] req@ffff9e35da718f00 x1874672085479040/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827900 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827875.337307:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672085479040/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827900 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827875.337319:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827875.337324:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787827875.337344:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085479040:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787827875.337354:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085479168:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787827875.337386:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787827875.337404:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827875/real 1787827875] req@ffff9e35da71a940 x1874672085479168/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827900 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827875.337418:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672085479168/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827900 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827875.337428:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827875.337431:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787827875.337434:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085479168:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787827875.337490:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085479296:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787827875.337510:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787827875.337520:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827875/real 1787827875] req@ffff9e35da7183c0 x1874672085479296/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827900 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827875.337554:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672085479296/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827900 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827875.337564:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827875.337568:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787827875.337573:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085479296:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787827875.463942:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787827875.463993:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787827875.464064:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787827876.488095:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787827877.511944:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787827878.536001:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787827880.456079:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085479424:192.168.203.141@tcp:250:kworker.0 00000800:00000100:2.0:1787827880.456187:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787827880.456255:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827880/real 1787827880] req@ffff9e35da718780 x1874672085479424/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827905 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827880.456317:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718780 x1874672085479424/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827905 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827880.456377:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827880.456386:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787827880.456395:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085479424:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1787827880.456411:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085479552:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787827880.456449:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787827880.456466:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827880/real 1787827880] req@ffff9e35da7183c0 x1874672085479552/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827905 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827880.456485:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672085479552/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827905 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827880.456492:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827880.456496:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787827880.456500:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085479552:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787827880.456529:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085479680:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787827880.456542:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787827880.456549:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827880/real 1787827880] req@ffff9e35da71a940 x1874672085479680/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827905 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827880.456558:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672085479680/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827905 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827880.456563:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827880.456565:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787827880.456567:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085479680:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787827880.456571:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085479808:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787827880.456592:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787827880.456599:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827880/real 1787827880] req@ffff9e35da718f00 x1874672085479808/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827905 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827880.456606:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672085479808/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827905 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827880.456616:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827880.456619:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787827880.456621:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085479808:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787827880.456625:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085479936:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787827880.456636:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787827880.456641:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827880/real 1787827880] req@ffff9e35da718b40 x1874672085479936/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827905 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827880.456648:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718b40 x1874672085479936/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827905 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827880.456655:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827880.456656:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787827880.456658:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085479936:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787827880.456663:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085480064:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787827880.456674:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787827880.456680:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827880/real 1787827880] req@ffff9e35da719a40 x1874672085480064/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827905 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827880.456688:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672085480064/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827905 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827880.456693:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827880.456695:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787827880.456697:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085480064:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787827880.456701:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085480192:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787827880.456711:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787827880.456717:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827880/real 1787827880] req@ffff9e35da71a1c0 x1874672085480192/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827905 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827880.456724:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a1c0 x1874672085480192/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827905 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827880.456737:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827880.456739:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787827880.456741:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085480192:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787827880.456746:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085480320:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787827880.456757:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787827880.456764:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827880/real 1787827880] req@ffff9e35da718000 x1874672085480320/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827905 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827880.456772:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718000 x1874672085480320/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827905 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827880.456778:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827880.456780:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787827880.456782:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085480320:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787827880.456785:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085480448:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787827880.456835:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787827880.456844:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827880/real 1787827880] req@ffff9e35da7192c0 x1874672085480448/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827905 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827880.456852:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7192c0 x1874672085480448/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827905 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827880.456858:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827880.456861:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787827880.456863:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085480448:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787827881.607885:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787827881.607969:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787827882.631944:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787827883.655932:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787827884.679917:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787827884.679966:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787827885.703920:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787827885.703995:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787827885.703998:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787827886.727914:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:3.0:1787827886.727925:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787827887.751927:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:3.0:1787827887.751927:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787827887.752026:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787827888.775896:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:3.0:1787827888.775907:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787827888.775991:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787827889.799924:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787827889.799988:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787827890.695907:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827890.695914:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787827890.695919:0:8:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827890.695927:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1326 00000100:00080000:3.0:1787827890.695949:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827890.695955:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787827890.695988:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649ad00 x1874672085481728/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827890.696009:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827890.696012:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827890.696013:0:8:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827890.696017:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1326 00000100:00080000:3.0:1787827890.696019:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827890.696023:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827890.696032:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827890.696043:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b0c0 x1874672085481856/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827890.696054:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827890.696056:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787827890.696058:0:8:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827890.696061:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1326 00000100:00080000:3.0:1787827890.696063:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827890.696065:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827890.696068:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827890.696076:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b480 x1874672085481984/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827890.696082:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827890.696084:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787827890.696085:0:8:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827890.696088:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1326 00000100:00080000:3.0:1787827890.696091:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827890.696093:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827890.696096:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:3.0:1787827890.696107:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499e00 x1874672085482112/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827890.696115:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827890.696118:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787827890.696119:0:8:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827890.696122:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1326 00000100:00080000:3.0:1787827890.696124:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827890.696127:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827890.696130:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:3.0:1787827890.696140:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c64983c0 x1874672085482240/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827890.696159:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827890.696162:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827890.696166:0:8:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827890.696175:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1326 00000100:00080000:3.0:1787827890.696179:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827890.696188:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827890.696195:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827890.696207:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a1c0 x1874672085482368/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827890.696213:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827890.696216:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787827890.696217:0:8:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827890.696223:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1326 00000100:00080000:3.0:1787827890.696227:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827890.696229:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827890.696234:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827890.696246:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498000 x1874672085482496/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827890.696255:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827890.696260:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787827890.696263:0:8:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827890.696272:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1326 00000100:00080000:3.0:1787827890.696276:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827890.696283:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827890.696290:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:3.0:1787827890.696301:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c64992c0 x1874672085482624/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827890.696319:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827890.696322:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787827890.696324:0:8:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827890.696328:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1326 00000100:00080000:3.0:1787827890.696331:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827890.696338:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827890.696343:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:3.0:1787827890.696356:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499a40 x1874672085482752/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:3.0:1787827890.823918:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787827890.823958:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:3.0:1787827890.823965:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:3.0:1787827891.847894:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787827891.847996:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827892.871881:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787827892.871985:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787827893.895932:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787827893.895965:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827894.919937:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787827895.943881:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787827895.943920:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827896.967950:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787827896.968022:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787827897.991913:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787827897.991982:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787827899.015906:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787827899.016001:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787827900.039924:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787827900.039984:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787827900.936089:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085484032:192.168.203.141@tcp:250:kworker.0 00000800:00000100:2.0:1787827900.936193:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787827900.936264:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827900/real 1787827900] req@ffff9e35da6f5e00 x1874672085484032/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827925 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827900.936322:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5e00 x1874672085484032/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827925 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827900.936357:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827900.936370:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787827900.936379:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085484032:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1787827900.936399:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085484160:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787827900.936448:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787827900.936466:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827900/real 1787827900] req@ffff9e35da6f4f00 x1874672085484160/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827925 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827900.936476:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4f00 x1874672085484160/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827925 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827900.936484:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827900.936486:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787827900.936489:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085484160:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787827900.936518:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085484288:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787827900.936532:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787827900.936539:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827900/real 1787827900] req@ffff9e35da6f5680 x1874672085484288/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827925 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827900.936547:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5680 x1874672085484288/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827925 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827900.936558:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827900.936560:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787827900.936563:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085484288:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787827900.936567:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085484416:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787827900.936577:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787827900.936583:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827900/real 1787827900] req@ffff9e35da6f6580 x1874672085484416/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827925 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827900.936591:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6580 x1874672085484416/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827925 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827900.936596:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827900.936598:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787827900.936600:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085484416:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787827900.936604:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085484544:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787827900.936614:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787827900.936620:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827900/real 1787827900] req@ffff9e35da6f6940 x1874672085484544/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827925 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827900.936628:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6940 x1874672085484544/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827925 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827900.936634:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827900.936635:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787827900.936638:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085484544:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787827900.936642:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085484672:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787827900.936654:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787827900.936667:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827900/real 1787827900] req@ffff9e35da6f4000 x1874672085484672/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827925 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827900.936675:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4000 x1874672085484672/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827925 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827900.936681:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827900.936683:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787827900.936685:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085484672:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787827900.936689:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085484800:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787827900.936701:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787827900.936707:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827900/real 1787827900] req@ffff9e35da6f6d00 x1874672085484800/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827925 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827900.936714:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6d00 x1874672085484800/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827925 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827900.936720:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827900.936721:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787827900.936723:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085484800:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787827900.936727:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085484928:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787827900.936738:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787827900.936744:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827900/real 1787827900] req@ffff9e35da6f70c0 x1874672085484928/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827925 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827900.936752:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f70c0 x1874672085484928/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827925 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827900.936758:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827900.936759:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787827900.936761:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085484928:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787827900.936765:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085485056:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787827900.936775:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787827900.936781:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827900/real 1787827900] req@ffff9e35da6f5a40 x1874672085485056/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827925 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827900.936788:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5a40 x1874672085485056/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827925 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827900.936794:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827900.936796:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787827900.936829:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085485056:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787827901.063934:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787827902.087937:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787827902.087947:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787827903.111902:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787827903.111984:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787827904.135906:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787827904.135968:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787827905.159891:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787827905.159945:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787827906.056049:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085485184:192.168.203.141@tcp:250:kworker.0 00000800:00000100:2.0:1787827906.056158:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787827906.056229:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827906/real 1787827906] req@ffff9e35da6f7480 x1874672085485184/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827931 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827906.056256:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7480 x1874672085485184/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827931 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827906.056301:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827906.056308:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787827906.056317:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085485184:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1787827906.056328:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085485312:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787827906.056359:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787827906.056371:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827906/real 1787827906] req@ffff9e35da6f61c0 x1874672085485312/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827931 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827906.056380:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f61c0 x1874672085485312/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827931 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827906.056387:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827906.056390:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787827906.056392:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085485312:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787827906.056425:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085485440:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787827906.056447:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787827906.056459:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827906/real 1787827906] req@ffff9e35da6f7840 x1874672085485440/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827931 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827906.056474:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7840 x1874672085485440/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827931 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827906.056480:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827906.056482:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787827906.056485:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085485440:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787827906.056489:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085485568:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787827906.056506:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787827906.056513:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827906/real 1787827906] req@ffff9e35da6f4780 x1874672085485568/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827931 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827906.056521:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4780 x1874672085485568/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827931 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827906.056527:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827906.056529:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787827906.056531:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085485568:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787827906.056536:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085485696:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787827906.056546:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787827906.056552:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827906/real 1787827906] req@ffff9e35da6f43c0 x1874672085485696/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827931 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827906.056559:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f43c0 x1874672085485696/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827931 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827906.056570:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827906.056572:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787827906.056575:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085485696:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787827906.056578:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085485824:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787827906.056588:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787827906.056594:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827906/real 1787827906] req@ffff9e35da6f4b40 x1874672085485824/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827931 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827906.056602:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4b40 x1874672085485824/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827931 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827906.056608:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827906.056610:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787827906.056611:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085485824:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787827906.056615:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085485952:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787827906.056626:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787827906.056632:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827906/real 1787827906] req@ffff9e35da6f5a40 x1874672085485952/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827931 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827906.056640:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5a40 x1874672085485952/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827931 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827906.056649:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827906.056652:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787827906.056656:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085485952:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787827906.056663:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085486080:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787827906.056684:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787827906.056698:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827906/real 1787827906] req@ffff9e35da6f70c0 x1874672085486080/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827931 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827906.056711:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f70c0 x1874672085486080/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827931 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827906.056720:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827906.056723:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787827906.056727:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085486080:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787827906.056734:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085486208:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787827906.056759:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787827906.056768:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827906/real 1787827906] req@ffff9e35da6f6d00 x1874672085486208/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827931 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827906.056776:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6d00 x1874672085486208/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827931 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827906.056782:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827906.056785:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787827906.056787:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085486208:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787827906.183942:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787827906.183995:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787827907.207973:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787827907.208011:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787827908.231922:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787827908.232017:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787827909.255936:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787827910.279968:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787827910.280023:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787827911.176062:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085486336:192.168.203.141@tcp:250:kworker.0 00000800:00000100:2.0:1787827911.176168:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787827911.176226:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827911/real 1787827911] req@ffff9e35da6f4000 x1874672085486336/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827936 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827911.176282:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4000 x1874672085486336/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827936 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827911.176303:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827911.176311:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787827911.176316:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085486336:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1787827911.176326:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085486464:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787827911.176355:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787827911.176369:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827911/real 1787827911] req@ffff9e35da6f6940 x1874672085486464/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827936 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827911.176378:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6940 x1874672085486464/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827936 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827911.176385:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827911.176388:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787827911.176390:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085486464:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787827911.176395:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085486592:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787827911.176407:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787827911.176413:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827911/real 1787827911] req@ffff9e35da6f6580 x1874672085486592/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827936 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827911.176421:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6580 x1874672085486592/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827936 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827911.176426:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827911.176429:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787827911.176431:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085486592:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787827911.176460:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085486720:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787827911.176471:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787827911.176477:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827911/real 1787827911] req@ffff9e35da6f5680 x1874672085486720/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827936 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827911.176485:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5680 x1874672085486720/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827936 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827911.176490:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827911.176492:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787827911.176494:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085486720:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787827911.176498:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085486848:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787827911.176508:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787827911.176514:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827911/real 1787827911] req@ffff9e35da6f4f00 x1874672085486848/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827936 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827911.176526:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4f00 x1874672085486848/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827936 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827911.176532:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827911.176534:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787827911.176535:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085486848:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787827911.176539:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085486976:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787827911.176549:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787827911.176556:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827911/real 1787827911] req@ffff9e35da6f5e00 x1874672085486976/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827936 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827911.176563:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5e00 x1874672085486976/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827936 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827911.176569:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827911.176570:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787827911.176572:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085486976:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787827911.176576:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085487104:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787827911.176586:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787827911.176592:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827911/real 1787827911] req@ffff9e35da6f6d00 x1874672085487104/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827936 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827911.176599:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6d00 x1874672085487104/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827936 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827911.176609:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827911.176611:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787827911.176613:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085487104:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787827911.176617:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085487232:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787827911.176626:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787827911.176632:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827911/real 1787827911] req@ffff9e35da6f70c0 x1874672085487232/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827936 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827911.176639:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f70c0 x1874672085487232/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827936 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827911.176645:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827911.176647:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787827911.176649:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085487232:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787827911.176652:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085487360:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787827911.176661:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787827911.176667:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827911/real 1787827911] req@ffff9e35da6f5a40 x1874672085487360/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827936 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827911.176675:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5a40 x1874672085487360/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827936 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827911.176680:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827911.176682:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787827911.176684:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085487360:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787827911.303969:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787827911.303982:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787827913.351916:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787827913.351941:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787827913.352061:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787827914.375941:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787827914.375995:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787827914.376010:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787827915.399942:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787827915.400036:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787827916.296039:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085487488:192.168.203.141@tcp:250:kworker.0 00000800:00000100:2.0:1787827916.296104:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787827916.296146:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827916/real 1787827916] req@ffff9e35da71a580 x1874672085487488/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827941 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827916.296178:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672085487488/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827941 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827916.296201:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827916.296209:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787827916.296215:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085487488:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1787827916.296244:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085487616:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787827916.296272:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787827916.296286:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827916/real 1787827916] req@ffff9e35da7192c0 x1874672085487616/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827941 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827916.296311:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7192c0 x1874672085487616/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827941 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827916.296329:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827916.296334:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787827916.296340:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085487616:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787827916.296352:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085487744:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787827916.296368:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787827916.296377:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827916/real 1787827916] req@ffff9e35da718000 x1874672085487744/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827941 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827916.296385:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718000 x1874672085487744/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827941 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827916.296391:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827916.296393:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787827916.296395:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085487744:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787827916.296399:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085487872:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787827916.296409:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787827916.296415:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827916/real 1787827916] req@ffff9e35da71a1c0 x1874672085487872/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827941 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827916.296422:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a1c0 x1874672085487872/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827941 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827916.296428:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827916.296429:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787827916.296431:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085487872:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787827916.296435:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085488000:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787827916.296445:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787827916.296451:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827916/real 1787827916] req@ffff9e35da719a40 x1874672085488000/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827941 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827916.296458:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672085488000/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827941 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827916.296464:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827916.296465:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787827916.296467:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085488000:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787827916.296478:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085488128:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787827916.296490:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787827916.296496:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827916/real 1787827916] req@ffff9e35da718b40 x1874672085488128/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827941 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827916.296503:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718b40 x1874672085488128/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827941 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827916.296509:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827916.296511:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787827916.296513:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085488128:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787827916.296527:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085488256:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787827916.296538:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787827916.296544:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827916/real 1787827916] req@ffff9e35da718f00 x1874672085488256/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827941 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827916.296556:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672085488256/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827941 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827916.296561:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827916.296563:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787827916.296565:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085488256:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787827916.296568:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085488384:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787827916.296578:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787827916.296583:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827916/real 1787827916] req@ffff9e35da71a940 x1874672085488384/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827941 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827916.296591:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672085488384/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827941 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827916.296596:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827916.296599:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787827916.296602:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085488384:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787827916.296608:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085488512:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787827916.296635:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787827916.296647:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827916/real 1787827916] req@ffff9e35da7183c0 x1874672085488512/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827941 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827916.296659:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672085488512/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827941 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827916.296670:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827916.296672:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787827916.296674:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085488512:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787827916.423940:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787827916.423956:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787827917.447878:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787827917.447939:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787827918.471887:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787827918.471910:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787827918.471983:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787827919.495895:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787827920.519884:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787827920.519942:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787827920.520001:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787827921.416022:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085488640:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1787827921.416125:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085488768:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787827921.416159:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085488896:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787827921.416187:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085489024:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787827921.416222:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085489152:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787827921.416252:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085489280:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787827921.416283:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085489408:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1787827921.543949:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787827922.567896:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787827922.567915:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787827923.591942:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787827923.592004:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787827923.592004:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1787827924.488085:0:2396:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.141:988 00000400:00000100:2.0:1787827924.488129:0:2396:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.141@tcp at host 192.168.203.141 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1787827924.488148:0:2396:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:2.0:1787827924.488198:0:2396:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000800:00000100:2.0:1787827924.488216:0:2396:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:2.0:1787827924.488233:0:2396:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:2.0:1787827924.488261:0:2396:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:2.0:1787827924.488275:0:2396:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:2.0:1787827924.488288:0:2396:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:2.0:1787827924.488302:0:2396:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:2.0:1787827924.488317:0:2396:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:2.0:1787827924.615960:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787827924.615991:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827925.639970:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787827925.640058:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787827926.536185:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085489792:192.168.203.141@tcp:250:kworker.0 00000800:00000100:2.0:1787827926.536295:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787827926.536360:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827926/real 1787827926] req@ffff9e35da6f4b40 x1874672085489792/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827951 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827926.536403:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4b40 x1874672085489792/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827951 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827926.536461:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827926.536474:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787827926.536499:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085489792:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1787827926.536574:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085489920:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787827926.536640:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787827926.536681:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827926/real 1787827926] req@ffff9e35da6f43c0 x1874672085489920/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827951 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827926.536755:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f43c0 x1874672085489920/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827951 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827926.536786:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827926.536862:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787827926.536871:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085489920:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787827926.536896:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085490048:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787827926.536961:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787827926.537004:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827926/real 1787827926] req@ffff9e35da6f4780 x1874672085490048/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827951 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827926.537035:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4780 x1874672085490048/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827951 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827926.537067:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827926.537074:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787827926.537081:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085490048:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787827926.537093:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085490176:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787827926.537128:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787827926.537148:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827926/real 1787827926] req@ffff9e35da6f7840 x1874672085490176/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827951 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827926.537167:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7840 x1874672085490176/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827951 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827926.537181:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827926.537186:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787827926.537191:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085490176:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787827926.537201:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085490304:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787827926.537232:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787827926.537255:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827926/real 1787827926] req@ffff9e35da6f61c0 x1874672085490304/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827951 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827926.537274:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f61c0 x1874672085490304/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827951 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827926.537287:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827926.537291:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787827926.537296:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085490304:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787827926.537328:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085490432:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787827926.537360:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787827926.537378:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827926/real 1787827926] req@ffff9e35da6f7480 x1874672085490432/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827951 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827926.537396:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7480 x1874672085490432/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827951 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827926.537409:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827926.537413:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787827926.537419:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085490432:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787827926.537428:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085490560:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787827926.537459:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787827926.537477:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827926/real 1787827926] req@ffff9e35da6f4b40 x1874672085490560/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827951 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827926.537497:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4b40 x1874672085490560/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827951 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827926.537511:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827926.537516:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787827926.537521:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085490560:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787827926.537531:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085490688:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787827926.537561:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787827926.537579:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827926/real 1787827926] req@ffff9e35da6f5a40 x1874672085490688/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827951 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827926.537598:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5a40 x1874672085490688/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827951 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827926.537612:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827926.537617:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787827926.537623:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085490688:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787827926.537640:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085490816:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787827926.537672:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787827926.537712:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827926/real 1787827926] req@ffff9e35da6f70c0 x1874672085490816/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827951 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827926.537732:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f70c0 x1874672085490816/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827951 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827926.537745:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827926.537750:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787827926.537755:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085490816:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787827926.663946:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787827926.664061:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787827927.687943:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787827927.688027:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787827928.711938:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787827929.735959:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787827929.736079:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827930.759950:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787827930.760055:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827931.656005:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827931.656013:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827931.656017:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827931.656026:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1367 00000100:00080000:2.0:1787827931.656037:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827931.656043:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787827931.656078:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f480 x1874672085490944/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827931.656104:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827931.656108:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827931.656110:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827931.656113:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1367 00000100:00080000:2.0:1787827931.656116:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827931.656120:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787827931.656129:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827931.656140:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f0c0 x1874672085491072/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827931.656149:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827931.656155:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787827931.656157:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827931.656169:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1367 00000100:00080000:2.0:1787827931.656174:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827931.656178:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787827931.656185:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827931.656202:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f840 x1874672085491200/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827931.656221:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827931.656226:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787827931.656228:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827931.656234:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1367 00000100:00080000:2.0:1787827931.656239:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827931.656246:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787827931.656256:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:2.0:1787827931.656275:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cb40 x1874672085491328/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827931.656295:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827931.656299:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787827931.656303:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827931.656308:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1367 00000100:00080000:2.0:1787827931.656316:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827931.656352:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787827931.656362:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:2.0:1787827931.656388:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c000 x1874672085491456/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827931.656425:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827931.656432:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827931.656437:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827931.656442:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1367 00000100:00080000:2.0:1787827931.656445:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827931.656449:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787827931.656453:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827931.656461:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e580 x1874672085491584/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827931.656491:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827931.656497:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787827931.656499:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827931.656505:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1367 00000100:00080000:2.0:1787827931.656509:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827931.656514:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787827931.656519:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827931.656558:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f480 x1874672085491712/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827931.656574:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827931.656582:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787827931.656588:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827931.656603:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1367 00000100:00080000:2.0:1787827931.656610:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827931.656622:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787827931.656634:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:2.0:1787827931.656677:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e1c0 x1874672085491840/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827931.656719:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827931.656724:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787827931.656726:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827931.656733:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1367 00000100:00080000:2.0:1787827931.656739:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827931.656746:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787827931.656757:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:2.0:1787827931.656777:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cf00 x1874672085491968/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787827931.783930:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787827932.807918:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827933.832035:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787827934.856010:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787827935.879933:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787827935.879995:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787827936.903934:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787827936.903977:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827936.904000:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787827937.927871:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787827937.927873:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787827938.951912:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:3.0:1787827938.951916:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787827938.951987:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787827939.975944:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:3.0:1787827939.975974:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787827939.976130:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787827940.999873:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:3.0:1787827940.999882:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787827942.023967:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787827942.024012:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787827943.047936:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787827943.048030:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827943.048066:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:3.0:1787827944.071891:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787827944.071894:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787827945.095904:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787827945.095966:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787827946.119942:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787827946.119991:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787827947.015975:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827947.015987:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787827947.015994:0:8:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827947.016021:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1382 00000100:00080000:3.0:1787827947.016030:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827947.016036:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787827947.016069:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498f00 x1874672085494400/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827947.016093:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827947.016096:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827947.016097:0:8:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827947.016100:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1382 00000100:00080000:3.0:1787827947.016102:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827947.016104:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827947.016113:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827947.016120:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b840 x1874672085494528/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827947.016127:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827947.016129:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787827947.016131:0:8:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827947.016134:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1382 00000100:00080000:3.0:1787827947.016136:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827947.016138:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827947.016141:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827947.016148:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649bc00 x1874672085494656/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827947.016154:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827947.016157:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787827947.016158:0:8:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827947.016161:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1382 00000100:00080000:3.0:1787827947.016163:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827947.016167:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827947.016171:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:3.0:1787827947.016177:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a940 x1874672085494784/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827947.016185:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827947.016188:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787827947.016190:0:8:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827947.016199:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1382 00000100:00080000:3.0:1787827947.016210:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827947.016212:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827947.016215:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:3.0:1787827947.016222:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498b40 x1874672085494912/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827947.016229:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827947.016232:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827947.016233:0:8:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827947.016236:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1382 00000100:00080000:3.0:1787827947.016238:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827947.016240:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827947.016243:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827947.016253:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498780 x1874672085495040/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827947.016259:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827947.016262:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787827947.016263:0:8:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827947.016267:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1382 00000100:00080000:3.0:1787827947.016269:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827947.016291:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827947.016298:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827947.016317:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499680 x1874672085495168/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827947.016361:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827947.016367:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787827947.016370:0:8:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827947.016381:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1382 00000100:00080000:3.0:1787827947.016397:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827947.016425:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827947.016440:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:3.0:1787827947.016460:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a580 x1874672085495296/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827947.016510:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827947.016515:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787827947.016518:0:8:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827947.016583:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1382 00000100:00080000:3.0:1787827947.016594:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827947.016604:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827947.016614:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:3.0:1787827947.016654:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499a40 x1874672085495424/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787827947.143948:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787827947.144005:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787827947.144006:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787827948.167974:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787827948.168101:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787827949.191897:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787827949.191962:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787827949.192008:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787827950.215947:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787827950.215994:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787827951.239960:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787827951.240010:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827951.240035:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787827952.263975:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787827952.263978:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787827952.264088:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827953.288004:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787827953.288028:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787827953.288126:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787827954.312003:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:3.0:1787827954.312027:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787827954.312208:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787827955.336000:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:3.0:1787827955.336002:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787827955.336142:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787827956.359866:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787827956.359910:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787827957.383864:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:3.0:1787827957.383872:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787827957.383949:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787827958.407965:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787827958.407995:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787827958.408004:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827959.431934:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787827959.432027:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787827960.455964:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787827960.456038:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827961.479964:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827962.375959:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827962.375971:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827962.375976:0:8:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827962.375989:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1397 00000100:00080000:2.0:1787827962.376001:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827962.376010:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787827962.376054:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c780 x1874672085497856/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827962.376095:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827962.376103:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827962.376107:0:8:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827962.376114:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1397 00000100:00080000:2.0:1787827962.376120:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827962.376127:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787827962.376144:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827962.376170:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498fc00 x1874672085497984/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827962.376218:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827962.376222:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787827962.376225:0:8:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827962.376232:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1397 00000100:00080000:2.0:1787827962.376237:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827962.376242:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787827962.376250:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827962.376268:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498de00 x1874672085498112/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827962.376280:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827962.376287:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787827962.376293:0:8:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827962.376300:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1397 00000100:00080000:2.0:1787827962.376305:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827962.376310:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787827962.376316:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:2.0:1787827962.376330:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498ed00 x1874672085498240/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827962.376342:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827962.376349:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787827962.376355:0:8:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827962.376367:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1397 00000100:00080000:2.0:1787827962.376377:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827962.376389:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787827962.376402:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:2.0:1787827962.376420:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c3c0 x1874672085498368/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827962.376477:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827962.376490:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827962.376496:0:8:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827962.376508:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1397 00000100:00080000:2.0:1787827962.376514:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827962.376519:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787827962.376524:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827962.376543:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e940 x1874672085498496/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827962.376562:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827962.376567:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787827962.376570:0:8:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827962.376576:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1397 00000100:00080000:2.0:1787827962.376581:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827962.376590:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787827962.376596:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827962.376613:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498d680 x1874672085498624/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827962.376662:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827962.376673:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787827962.376676:0:8:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827962.376682:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1397 00000100:00080000:2.0:1787827962.376689:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827962.376707:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787827962.376721:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:2.0:1787827962.376738:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c780 x1874672085498752/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827962.376768:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827962.376779:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787827962.376784:0:8:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827962.376790:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1397 00000100:00080000:2.0:1787827962.376795:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827962.376856:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787827962.376863:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:2.0:1787827962.376878:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cf00 x1874672085498880/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787827962.504025:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787827962.504205:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827963.528015:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787827963.528103:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827964.551883:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787827964.551952:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827965.576102:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787827966.599945:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787827966.600020:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787827967.495951:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827967.495968:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787827967.495975:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827967.495990:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1402 00000100:00080000:3.0:1787827967.496007:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827967.496019:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787827967.496084:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c64992c0 x1874672085499008/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827967.496122:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827967.496136:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827967.496140:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827967.496147:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1402 00000100:00080000:3.0:1787827967.496155:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827967.496162:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827967.496179:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827967.496225:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498000 x1874672085499136/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827967.496253:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827967.496260:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787827967.496263:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827967.496270:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1402 00000100:00080000:3.0:1787827967.496276:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827967.496284:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827967.496295:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827967.496324:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a1c0 x1874672085499264/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827967.496347:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827967.496354:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787827967.496358:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827967.496372:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1402 00000100:00080000:3.0:1787827967.496387:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827967.496427:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827967.496442:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:3.0:1787827967.496468:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c64983c0 x1874672085499392/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827967.496484:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827967.496489:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787827967.496491:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827967.496498:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1402 00000100:00080000:3.0:1787827967.496507:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827967.496518:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827967.496525:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:3.0:1787827967.496557:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499e00 x1874672085499520/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827967.496574:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827967.496582:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827967.496586:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827967.496598:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1402 00000100:00080000:3.0:1787827967.496607:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827967.496617:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827967.496622:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827967.496635:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b480 x1874672085499648/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827967.496652:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827967.496659:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787827967.496670:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827967.496682:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1402 00000100:00080000:3.0:1787827967.496694:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827967.496705:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827967.496716:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827967.496738:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b0c0 x1874672085499776/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827967.496761:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827967.496766:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787827967.496772:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827967.496784:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1402 00000100:00080000:3.0:1787827967.496789:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827967.496797:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827967.496843:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:3.0:1787827967.496859:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649ad00 x1874672085499904/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827967.496872:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827967.496876:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787827967.496879:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827967.496884:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1402 00000100:00080000:3.0:1787827967.496889:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827967.496900:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827967.496909:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:3.0:1787827967.496929:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c64992c0 x1874672085500032/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787827967.623938:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787827967.624048:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787827968.647931:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787827968.648004:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787827969.671945:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787827969.672077:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827970.695974:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787827970.696069:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787827971.719992:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787827971.720076:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787827972.615957:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827972.615970:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787827972.615976:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827972.615990:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1407 00000100:00080000:3.0:1787827972.616001:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827972.616009:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787827972.616051:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499a40 x1874672085500160/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827972.616086:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827972.616090:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827972.616092:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827972.616101:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1407 00000100:00080000:3.0:1787827972.616105:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827972.616109:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827972.616120:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827972.616134:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a580 x1874672085500288/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827972.616146:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827972.616149:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787827972.616152:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827972.616155:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1407 00000100:00080000:3.0:1787827972.616158:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827972.616162:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827972.616168:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827972.616182:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499680 x1874672085500416/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827972.616206:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827972.616210:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787827972.616212:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827972.616216:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1407 00000100:00080000:3.0:1787827972.616219:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827972.616223:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827972.616227:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:3.0:1787827972.616239:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498780 x1874672085500544/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827972.616250:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827972.616253:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787827972.616255:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827972.616259:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1407 00000100:00080000:3.0:1787827972.616263:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827972.616266:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827972.616271:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:3.0:1787827972.616287:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498b40 x1874672085500672/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827972.616299:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827972.616303:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827972.616305:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827972.616309:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1407 00000100:00080000:3.0:1787827972.616313:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827972.616318:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827972.616323:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827972.616337:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a940 x1874672085500800/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827972.616352:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827972.616356:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787827972.616357:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827972.616411:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1407 00000100:00080000:3.0:1787827972.616417:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827972.616430:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827972.616441:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827972.616508:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649bc00 x1874672085500928/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827972.616525:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827972.616534:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787827972.616554:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827972.616560:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1407 00000100:00080000:3.0:1787827972.616565:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827972.616571:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827972.616578:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:3.0:1787827972.616595:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b840 x1874672085501056/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827972.616609:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827972.616616:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787827972.616618:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827972.616623:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1407 00000100:00080000:3.0:1787827972.616629:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827972.616635:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827972.616640:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:3.0:1787827972.616657:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498f00 x1874672085501184/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787827972.743978:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787827972.744048:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787827973.767952:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:3.0:1787827973.767972:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787827973.768113:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787827974.791907:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787827974.792018:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827975.815903:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787827975.815994:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787827976.839950:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787827976.839967:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827977.735898:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827977.735908:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827977.735914:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827977.735939:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1413 00000100:00080000:2.0:1787827977.735951:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827977.735959:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787827977.736008:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e1c0 x1874672085501312/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827977.736033:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827977.736037:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827977.736039:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827977.736042:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1413 00000100:00080000:2.0:1787827977.736046:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827977.736049:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787827977.736058:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827977.736078:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f480 x1874672085501440/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827977.736087:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827977.736090:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787827977.736091:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827977.736094:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1413 00000100:00080000:2.0:1787827977.736097:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827977.736100:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787827977.736103:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827977.736111:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e580 x1874672085501568/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827977.736121:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827977.736123:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787827977.736141:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827977.736159:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1413 00000100:00080000:2.0:1787827977.736162:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827977.736171:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787827977.736179:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:2.0:1787827977.736196:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c000 x1874672085501696/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827977.736210:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827977.736215:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787827977.736218:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827977.736224:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1413 00000100:00080000:2.0:1787827977.736230:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827977.736241:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787827977.736249:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:2.0:1787827977.736263:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cb40 x1874672085501824/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827977.736282:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827977.736288:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827977.736294:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827977.736309:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1413 00000100:00080000:2.0:1787827977.736316:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827977.736342:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787827977.736357:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827977.736393:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f840 x1874672085501952/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827977.736407:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827977.736411:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787827977.736413:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827977.736418:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1413 00000100:00080000:2.0:1787827977.736422:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827977.736427:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787827977.736432:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827977.736449:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f0c0 x1874672085502080/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827977.736486:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827977.736516:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787827977.736523:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827977.736537:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1413 00000100:00080000:2.0:1787827977.736548:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827977.736558:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787827977.736565:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:2.0:1787827977.736587:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f480 x1874672085502208/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827977.736613:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827977.736629:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787827977.736634:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827977.736647:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1413 00000100:00080000:2.0:1787827977.736658:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827977.736669:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787827977.736678:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:2.0:1787827977.736698:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e1c0 x1874672085502336/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787827977.863897:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787827977.863936:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787827978.887972:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787827978.888138:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787827979.911992:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787827979.912003:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827980.935923:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787827980.936026:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787827981.959938:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787827981.959956:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787827982.855953:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827982.855968:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787827982.855976:0:8:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827982.855992:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1418 00000100:00080000:3.0:1787827982.856007:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827982.856019:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787827982.856081:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498f00 x1874672085502464/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827982.856113:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827982.856118:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827982.856120:0:8:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827982.856126:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1418 00000100:00080000:3.0:1787827982.856130:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827982.856135:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827982.856163:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827982.856182:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b840 x1874672085502592/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827982.856202:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827982.856206:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787827982.856225:0:8:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827982.856233:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1418 00000100:00080000:3.0:1787827982.856241:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827982.856248:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827982.856257:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827982.856276:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649bc00 x1874672085502720/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827982.856322:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827982.856330:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787827982.856335:0:8:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827982.856346:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1418 00000100:00080000:3.0:1787827982.856353:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827982.856360:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827982.856370:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:3.0:1787827982.856395:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a940 x1874672085502848/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827982.856470:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827982.856475:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787827982.856478:0:8:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827982.856489:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1418 00000100:00080000:3.0:1787827982.856497:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827982.856539:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827982.856555:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:3.0:1787827982.856599:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498b40 x1874672085502976/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827982.856620:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827982.856628:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827982.856636:0:8:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827982.856646:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1418 00000100:00080000:3.0:1787827982.856651:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827982.856658:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827982.856667:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827982.856689:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498780 x1874672085503104/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827982.856714:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827982.856723:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787827982.856746:0:8:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827982.856758:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1418 00000100:00080000:3.0:1787827982.856765:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827982.856777:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827982.856789:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827982.856833:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499680 x1874672085503232/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827982.856903:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827982.856910:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787827982.856914:0:8:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827982.856927:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1418 00000100:00080000:3.0:1787827982.856937:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827982.856949:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827982.856957:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:3.0:1787827982.856985:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a580 x1874672085503360/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827982.857014:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827982.857022:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787827982.857029:0:8:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827982.857060:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1418 00000100:00080000:3.0:1787827982.857069:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827982.857079:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787827982.857088:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:3.0:1787827982.857127:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499a40 x1874672085503488/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787827982.983924:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:3.0:1787827982.984153:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787827982.984253:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787827984.008036:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787827984.008114:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787827985.031922:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787827985.032065:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787827986.055862:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787827986.055952:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827987.079925:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787827987.079926:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787827988.103897:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787827988.103956:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787827989.127890:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787827990.151967:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787827990.152083:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1787827991.048184:0:2397:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.141:988 00000400:00000100:2.0:1787827991.048334:0:2397:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.141@tcp at host 192.168.203.141 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1787827991.048355:0:2397:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:2.0:1787827991.048422:0:2397:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000800:00000100:2.0:1787827991.048450:0:2397:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:2.0:1787827991.048472:0:2397:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:2.0:1787827991.048498:0:2397:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:2.0:1787827991.048545:0:2397:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:2.0:1787827991.048561:0:2397:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:2.0:1787827991.048583:0:2397:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:2.0:1787827991.048602:0:2397:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:2.0:1787827991.175927:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787827991.176113:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827992.199988:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00080000:2.0:1787827993.096048:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827993.096076:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827993.096081:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827993.096095:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1428 00000100:00080000:2.0:1787827993.096110:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827993.096120:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787827993.096176:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cf00 x1874672085504768/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827993.096201:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827993.096205:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827993.096208:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827993.096217:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1428 00000100:00080000:2.0:1787827993.096234:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827993.096240:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787827993.096254:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827993.096302:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c780 x1874672085504896/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827993.096329:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827993.096337:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787827993.096341:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827993.096354:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1428 00000100:00080000:2.0:1787827993.096362:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827993.096371:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787827993.096382:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827993.096426:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498d680 x1874672085505024/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827993.096463:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827993.096470:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787827993.096474:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827993.096489:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1428 00000100:00080000:2.0:1787827993.096499:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827993.096506:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787827993.096516:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:2.0:1787827993.096537:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e940 x1874672085505152/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827993.096557:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827993.096563:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787827993.096568:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827993.096576:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1428 00000100:00080000:2.0:1787827993.096580:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827993.096584:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787827993.096593:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:2.0:1787827993.096607:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c3c0 x1874672085505280/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827993.096618:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827993.096621:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827993.096623:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827993.096626:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1428 00000100:00080000:2.0:1787827993.096630:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827993.096632:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787827993.096636:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827993.096651:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498ed00 x1874672085505408/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827993.096660:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827993.096664:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787827993.096669:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827993.096679:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1428 00000100:00080000:2.0:1787827993.096687:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827993.096696:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787827993.096706:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827993.096725:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498de00 x1874672085505536/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827993.096742:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827993.096749:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787827993.096754:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827993.096766:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1428 00000100:00080000:2.0:1787827993.096775:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827993.096782:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787827993.096793:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:2.0:1787827993.096874:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498fc00 x1874672085505664/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827993.096894:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827993.096900:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787827993.096904:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827993.096916:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1428 00000100:00080000:2.0:1787827993.096927:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827993.096938:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787827993.096948:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:2.0:1787827993.096967:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cf00 x1874672085505792/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787827993.224112:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787827993.224187:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787827994.248006:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787827994.248151:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787827995.271948:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787827996.295962:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787827996.296004:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787827997.319932:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787827997.320034:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787827997.320154:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827998.215915:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827998.215945:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827998.215950:0:8:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827998.215960:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1433 00000100:00080000:2.0:1787827998.215970:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827998.215977:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787827998.216023:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e1c0 x1874672085505920/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827998.216060:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827998.216064:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827998.216066:0:8:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827998.216069:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1433 00000100:00080000:2.0:1787827998.216073:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827998.216076:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787827998.216102:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827998.216126:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f480 x1874672085506048/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827998.216136:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827998.216140:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787827998.216142:0:8:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827998.216146:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1433 00000100:00080000:2.0:1787827998.216149:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827998.216154:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787827998.216161:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827998.216175:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f0c0 x1874672085506176/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827998.216207:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827998.216211:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787827998.216213:0:8:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827998.216218:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1433 00000100:00080000:2.0:1787827998.216223:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827998.216228:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787827998.216244:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:2.0:1787827998.216256:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f840 x1874672085506304/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827998.216270:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827998.216278:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787827998.216281:0:8:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827998.216289:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1433 00000100:00080000:2.0:1787827998.216295:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827998.216319:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787827998.216328:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:2.0:1787827998.216359:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cb40 x1874672085506432/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827998.216404:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827998.216415:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827998.216421:0:8:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827998.216434:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1433 00000100:00080000:2.0:1787827998.216443:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827998.216452:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787827998.216458:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827998.216478:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c000 x1874672085506560/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827998.216494:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827998.216498:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787827998.216500:0:8:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827998.216510:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1433 00000100:00080000:2.0:1787827998.216516:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827998.216520:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787827998.216525:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827998.216547:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e580 x1874672085506688/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827998.216561:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827998.216569:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787827998.216574:0:8:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827998.216585:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1433 00000100:00080000:2.0:1787827998.216593:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827998.216600:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787827998.216606:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:2.0:1787827998.216624:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cf00 x1874672085506816/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827998.216638:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827998.216646:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787827998.216648:0:8:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827998.216653:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1433 00000100:00080000:2.0:1787827998.216656:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827998.216660:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787827998.216676:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:2.0:1787827998.216696:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498fc00 x1874672085506944/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787827998.344074:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787827998.344074:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787827999.367951:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787827999.368010:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827999.368040:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:3.0:1787827999.368073:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828000.391934:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828000.392083:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828000.392329:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828001.415876:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828001.415931:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828002.439946:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828002.440099:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828003.335913:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828003.335924:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828003.335929:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828003.335941:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1438 00000100:00080000:2.0:1787828003.335953:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828003.335961:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787828003.336003:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498de00 x1874672085507072/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828003.336036:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828003.336041:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828003.336043:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828003.336047:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1438 00000100:00080000:2.0:1787828003.336051:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828003.336054:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828003.336065:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828003.336078:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498ed00 x1874672085507200/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828003.336090:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828003.336094:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787828003.336096:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828003.336100:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1438 00000100:00080000:2.0:1787828003.336106:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828003.336110:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828003.336115:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828003.336127:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c3c0 x1874672085507328/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828003.336148:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828003.336175:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787828003.336177:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828003.336185:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1438 00000100:00080000:2.0:1787828003.336200:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828003.336211:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828003.336220:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:2.0:1787828003.336252:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e940 x1874672085507456/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828003.336266:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828003.336272:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787828003.336274:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828003.336281:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1438 00000100:00080000:2.0:1787828003.336285:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828003.336289:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828003.336295:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:2.0:1787828003.336309:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498d680 x1874672085507584/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828003.336325:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828003.336343:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828003.336348:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828003.336362:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1438 00000100:00080000:2.0:1787828003.336377:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828003.336386:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828003.336397:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828003.336419:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c780 x1874672085507712/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828003.336430:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828003.336434:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787828003.336437:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828003.336444:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1438 00000100:00080000:2.0:1787828003.336449:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828003.336455:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828003.336468:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828003.336500:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498fc00 x1874672085507840/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828003.336521:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828003.336536:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787828003.336550:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828003.336564:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1438 00000100:00080000:2.0:1787828003.336577:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828003.336589:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828003.336596:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:2.0:1787828003.336620:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cf00 x1874672085507968/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828003.336665:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828003.336668:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787828003.336672:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828003.336681:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1438 00000100:00080000:2.0:1787828003.336688:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828003.336693:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828003.336714:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:2.0:1787828003.336732:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e580 x1874672085508096/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828003.463937:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787828003.464008:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828004.487937:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828004.487969:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828005.511913:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828005.512065:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828006.536026:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828006.536178:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828007.559977:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:3.0:1787828007.560004:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828007.560148:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828008.455924:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828008.455940:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828008.455948:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828008.455964:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1443 00000100:00080000:2.0:1787828008.455983:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828008.455995:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787828008.456060:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c000 x1874672085508224/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828008.456105:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828008.456112:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828008.456143:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828008.456149:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1443 00000100:00080000:2.0:1787828008.456156:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828008.456162:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828008.456179:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828008.456199:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cb40 x1874672085508352/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828008.456219:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828008.456230:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787828008.456232:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828008.456240:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1443 00000100:00080000:2.0:1787828008.456244:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828008.456248:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828008.456254:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828008.456271:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f840 x1874672085508480/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828008.456284:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828008.456289:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787828008.456292:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828008.456297:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1443 00000100:00080000:2.0:1787828008.456301:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828008.456306:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828008.456314:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:2.0:1787828008.456327:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f0c0 x1874672085508608/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828008.456339:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828008.456344:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787828008.456346:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828008.456355:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1443 00000100:00080000:2.0:1787828008.456361:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828008.456369:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828008.456376:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:2.0:1787828008.456391:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f480 x1874672085508736/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828008.456403:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828008.456408:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828008.456411:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828008.456416:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1443 00000100:00080000:2.0:1787828008.456422:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828008.456428:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828008.456435:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828008.456459:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e1c0 x1874672085508864/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828008.456475:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828008.456482:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787828008.456488:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828008.456505:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1443 00000100:00080000:2.0:1787828008.456514:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828008.456521:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828008.456528:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828008.456567:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e580 x1874672085508992/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828008.456580:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828008.456585:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787828008.456587:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828008.456592:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1443 00000100:00080000:2.0:1787828008.456596:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828008.456601:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828008.456608:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:2.0:1787828008.456628:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cf00 x1874672085509120/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828008.456643:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828008.456648:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787828008.456651:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828008.456658:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1443 00000100:00080000:2.0:1787828008.456665:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828008.456674:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828008.456686:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:2.0:1787828008.456715:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498fc00 x1874672085509248/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787828008.583971:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828008.584006:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828008.584100:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828009.607939:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:3.0:1787828009.607972:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828009.608095:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828010.631969:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828010.632083:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828011.655887:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828011.656082:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828012.679985:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828012.680121:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828013.703941:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828013.704054:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828014.727944:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787828014.728013:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828014.728059:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828015.751935:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:3.0:1787828015.751967:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828015.752051:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828016.775903:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828016.776007:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828017.799898:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787828017.800062:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828018.696170:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828018.696185:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787828018.696192:0:8:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828018.696209:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1454 00000100:00080000:3.0:1787828018.696227:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828018.696239:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787828018.696304:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c64992c0 x1874672085510528/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828018.696362:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828018.696368:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828018.696370:0:8:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828018.696376:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1454 00000100:00080000:3.0:1787828018.696380:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828018.696385:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828018.696403:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828018.696421:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649ad00 x1874672085510656/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828018.696437:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828018.696449:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787828018.696455:0:8:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828018.696472:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1454 00000100:00080000:3.0:1787828018.696488:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828018.696511:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828018.696522:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828018.696550:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b0c0 x1874672085510784/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828018.696571:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828018.696580:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787828018.696583:0:8:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828018.696605:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1454 00000100:00080000:3.0:1787828018.696618:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828018.696625:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828018.696635:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:3.0:1787828018.696656:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b480 x1874672085510912/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828018.696669:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828018.696675:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787828018.696678:0:8:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828018.696692:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1454 00000100:00080000:3.0:1787828018.696702:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828018.696714:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828018.696728:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:3.0:1787828018.696749:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499e00 x1874672085511040/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828018.696851:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828018.696857:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828018.696860:0:8:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828018.696869:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1454 00000100:00080000:3.0:1787828018.696874:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828018.696881:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828018.696891:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828018.696919:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c64983c0 x1874672085511168/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828018.696952:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828018.696957:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787828018.696960:0:8:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828018.696968:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1454 00000100:00080000:3.0:1787828018.696972:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828018.696978:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828018.696984:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828018.697003:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a1c0 x1874672085511296/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828018.697021:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828018.697026:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787828018.697029:0:8:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828018.697039:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1454 00000100:00080000:3.0:1787828018.697075:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828018.697082:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828018.697090:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:3.0:1787828018.697128:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498000 x1874672085511424/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828018.697160:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828018.697174:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787828018.697183:0:8:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828018.697204:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1454 00000100:00080000:3.0:1787828018.697217:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828018.697230:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828018.697241:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:3.0:1787828018.697268:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c64992c0 x1874672085511552/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828018.823948:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828018.824138:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828019.847878:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828019.847931:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828020.871901:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:3.0:1787828020.871916:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787828020.872011:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828021.895996:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828021.896119:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828022.919891:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787828022.919966:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828023.943899:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828023.943979:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828024.967894:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828024.967959:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828025.991897:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828025.991919:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828025.991964:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828027.015914:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:3.0:1787828027.015917:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828027.016032:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828028.039964:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828028.040105:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828029.063912:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828029.063939:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828030.087941:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828030.088066:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828031.111991:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828031.112074:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828032.135974:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828033.160032:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787828033.160140:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828034.183976:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828034.184012:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828035.208093:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828035.208190:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828036.231942:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:3.0:1787828036.231967:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787828036.232083:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828037.256081:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:3.0:1787828037.256093:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828037.256284:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828038.279998:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:3.0:1787828038.280012:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828038.280113:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828039.303924:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828039.304006:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787828039.304093:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828040.327911:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828040.327976:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828041.352010:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787828041.352149:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828042.376044:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828042.376070:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828043.399985:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787828043.400095:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828044.296013:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828044.296024:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828044.296029:0:8:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828044.296042:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1479 00000100:00080000:2.0:1787828044.296054:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828044.296063:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787828044.296107:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c780 x1874672085516288/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828044.296149:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828044.296153:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828044.296155:0:8:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828044.296159:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1479 00000100:00080000:2.0:1787828044.296163:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828044.296168:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828044.296179:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828044.296203:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498d680 x1874672085516416/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828044.296215:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828044.296227:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787828044.296233:0:8:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828044.296255:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1479 00000100:00080000:2.0:1787828044.296260:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828044.296263:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828044.296270:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828044.296286:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e940 x1874672085516544/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828044.296297:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828044.296301:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787828044.296304:0:8:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828044.296308:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1479 00000100:00080000:2.0:1787828044.296320:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828044.296324:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828044.296329:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:2.0:1787828044.296341:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c3c0 x1874672085516672/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828044.296368:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828044.296373:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787828044.296376:0:8:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828044.296384:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1479 00000100:00080000:2.0:1787828044.296388:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828044.296391:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828044.296395:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:2.0:1787828044.296408:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498ed00 x1874672085516800/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828044.296418:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828044.296421:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828044.296423:0:8:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828044.296429:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1479 00000100:00080000:2.0:1787828044.296434:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828044.296447:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828044.296452:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828044.296466:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498de00 x1874672085516928/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828044.296549:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828044.296552:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787828044.296555:0:8:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828044.296568:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1479 00000100:00080000:2.0:1787828044.296575:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828044.296593:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828044.296618:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828044.296642:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498fc00 x1874672085517056/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828044.296652:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828044.296656:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787828044.296658:0:8:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828044.296662:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1479 00000100:00080000:2.0:1787828044.296665:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828044.296670:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828044.296681:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:2.0:1787828044.296701:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cf00 x1874672085517184/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828044.296716:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828044.296729:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787828044.296731:0:8:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828044.296734:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1479 00000100:00080000:2.0:1787828044.296741:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828044.296754:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828044.296766:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:2.0:1787828044.296795:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e580 x1874672085517312/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:3.0:1787828044.424062:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828044.424068:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828045.447953:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828045.447997:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787828046.472010:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828046.472058:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:3.0:1787828047.495958:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828047.496068:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828048.519962:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828048.520086:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828049.544017:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787828049.544082:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787828050.567991:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828050.568020:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828051.591934:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828051.592024:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828051.592135:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828052.615890:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787828052.615906:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787828052.615987:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828053.639936:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828053.639981:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828054.536207:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085518592:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1787828054.536378:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085518720:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828054.536470:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085518848:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828054.536532:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085518976:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828054.536626:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085519104:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828054.536700:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085519232:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828054.536929:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085519360:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828054.537050:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085519488:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828054.537170:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085519616:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828054.663908:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828054.663978:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828054.664048:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828055.687909:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787828055.687918:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787828055.688022:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828056.711888:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828056.711941:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828057.610210:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828054/real 1787828057] req@ffff9e35da7192c0 x1874672085519616/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828079 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828057.610300:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7192c0 x1874672085519616/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828079 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828057.610324:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828057.610336:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828057.610348:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085519616:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828057.610365:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828054/real 1787828057] req@ffff9e35da718000 x1874672085519488/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828079 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828057.610419:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718000 x1874672085519488/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828079 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828057.610434:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828057.610439:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828057.610444:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085519488:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828057.610453:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828054/real 1787828057] req@ffff9e35da71a1c0 x1874672085519360/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828079 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828057.610470:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a1c0 x1874672085519360/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828079 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828057.610481:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828057.610487:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828057.610494:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085519360:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828057.610508:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828054/real 1787828057] req@ffff9e35da719a40 x1874672085519232/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828079 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828057.610538:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672085519232/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828079 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828057.610556:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828057.610564:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828057.610571:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085519232:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828057.610584:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828054/real 1787828057] req@ffff9e35da718f00 x1874672085519104/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828079 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828057.610602:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672085519104/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828079 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828057.610613:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828057.610618:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828057.610625:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085519104:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828057.610638:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828054/real 1787828057] req@ffff9e35da718b40 x1874672085518976/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828079 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828057.610661:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718b40 x1874672085518976/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828079 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828057.610678:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828057.610684:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828057.610691:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085518976:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828057.610704:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828054/real 1787828057] req@ffff9e35da7183c0 x1874672085518848/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828079 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828057.610729:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672085518848/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828079 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828057.610745:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828057.610749:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828057.610754:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085518848:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828057.610764:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828054/real 1787828057] req@ffff9e35da71a940 x1874672085518720/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828079 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828057.610936:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672085518720/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828079 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828057.610955:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828057.610962:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828057.610969:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085518720:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828057.610984:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828054/real 1787828057] req@ffff9e35da71a580 x1874672085518592/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828079 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828057.611010:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672085518592/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828079 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828057.611035:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828057.611039:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828057.611044:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085518592:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1787828057.736044:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828057.736189:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787828058.760179:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828059.656065:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828059.656086:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787828059.656094:0:8:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828059.656113:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1495 00000100:00080000:3.0:1787828059.656133:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828059.656150:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787828059.656236:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499a40 x1874672085519744/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828059.656310:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828059.656319:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828059.656323:0:8:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828059.656332:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1495 00000100:00080000:3.0:1787828059.656341:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828059.656351:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828059.656392:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828059.656426:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a580 x1874672085519872/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828059.656443:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828059.656450:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787828059.656455:0:8:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828059.656465:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1495 00000100:00100000:2.0:1787828059.656487:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085519744:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1787828059.656491:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828059.656510:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828059.656528:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828059.656567:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499680 x1874672085520000/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828059.656593:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828059.656602:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787828059.656608:0:8:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1787828059.656624:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1787828059.656632:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1495 00000100:00080000:3.0:1787828059.656647:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828059.656658:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828059.656670:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:3.0:1787828059.656702:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498780 x1874672085520128/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828059.656719:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828059/real 1787828059] req@ffff9e35c6499a40 x1874672085519744/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828084 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828059.656740:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828059.656790:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:2.0:1787828059.656793:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499a40 x1874672085519744/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828084 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828059.656846:0:8:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828059.656862:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1495 00000100:00080000:3.0:1787828059.656873:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828059.656882:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828059.656894:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:3.0:1787828059.656936:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498b40 x1874672085520256/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828059.656941:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828059.656967:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828059.656993:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085519744:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1787828059.656997:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828059.657012:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828059.657019:0:8:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787828059.657023:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085519872:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1787828059.657039:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1495 00000800:00000100:2.0:1787828059.657085:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828059.657110:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828059/real 1787828059] req@ffff9e35c649a580 x1874672085519872/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828084 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828059.657134:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a580 x1874672085519872/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828084 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828059.657149:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828059.657157:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828059.657162:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085519872:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828059.657203:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085520000:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828059.657238:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828059.657256:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828059/real 1787828059] req@ffff9e35c6499680 x1874672085520000/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828084 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828059.657283:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499680 x1874672085520000/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828084 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828059.657294:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828059.657299:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828059.657304:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085520000:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828059.657314:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085520128:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828059.657345:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828059.657363:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828059/real 1787828059] req@ffff9e35c6498780 x1874672085520128/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828084 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828059.657382:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498780 x1874672085520128/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828084 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828059.657395:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828059.657400:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828059.657405:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085520128:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828059.657415:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085520256:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828059.657446:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828059.657463:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828059/real 1787828059] req@ffff9e35c6498b40 x1874672085520256/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828084 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828059.657480:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498b40 x1874672085520256/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828084 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828059.657500:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828059.657505:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828059.657509:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085520256:192.168.203.141@tcp:8:kworker.0 00000100:00080000:3.0:1787828059.658114:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828059.658130:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828059.658145:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828059.658181:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a940 x1874672085520384/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828059.658270:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085520384:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828059.658307:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1787828059.658387:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828059.658401:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:2.0:1787828059.658403:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828059/real 1787828059] req@ffff9e35c649a940 x1874672085520384/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828084 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828059.658409:0:8:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828059.658426:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1495 00000100:00080000:3.0:1787828059.658439:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828059.658449:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787828059.658458:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a940 x1874672085520384/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828084 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1787828059.658462:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1787828059.658493:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1787828059.658496:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649bc00 x1874672085520512/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828059.658504:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828059.658530:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085520384:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1787828059.658555:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828059.658564:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787828059.658567:0:8:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828059.658577:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1495 00000100:00100000:2.0:1787828059.658578:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085520512:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1787828059.658594:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828059.658604:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828059.658612:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000800:00000100:2.0:1787828059.658632:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828059.658634:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b840 x1874672085520640/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828059.658653:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787828059.658655:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828059/real 1787828059] req@ffff9e35c649bc00 x1874672085520512/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828084 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828059.658670:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787828059.658679:0:8:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828059.658703:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1495 00000100:00100000:2.0:1787828059.658712:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649bc00 x1874672085520512/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828084 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828059.658721:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828059.658738:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:2.0:1787828059.658748:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000008:00100000:3.0:1787828059.658753:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00080000:2.0:1787828059.658784:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828059.658824:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085520512:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787828059.658849:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498f00 x1874672085520768/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828059.658852:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085520640:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828059.658937:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828059.658962:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828059/real 1787828059] req@ffff9e35c649b840 x1874672085520640/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828084 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828059.658982:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b840 x1874672085520640/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828084 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828059.658996:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828059.659001:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828059.659004:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085520640:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828059.659021:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085520768:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828059.659044:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828059.659055:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828059/real 1787828059] req@ffff9e35c6498f00 x1874672085520768/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828084 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828059.659067:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498f00 x1874672085520768/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828084 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828059.659075:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828059.659078:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828059.659081:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085520768:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828059.784050:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828059.784119:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828060.807920:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828060.808021:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828061.832004:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828061.832044:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828062.855938:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828062.855971:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828063.879927:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828063.879996:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828064.776221:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085520896:192.168.203.141@tcp:250:kworker.0 00000800:00000100:2.0:1787828064.776305:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828064.776354:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828064/real 1787828064] req@ffff9e35da6f7480 x1874672085520896/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828089 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828064.776393:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7480 x1874672085520896/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828089 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828064.776427:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828064.776439:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828064.776447:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085520896:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1787828064.776464:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085521024:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828064.776548:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828064.776587:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828064/real 1787828064] req@ffff9e35da6f6d00 x1874672085521024/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828089 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828064.776656:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6d00 x1874672085521024/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828089 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828064.776682:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828064.776696:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828064.776703:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085521024:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828064.776799:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085521152:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828064.776881:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828064.776901:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828064/real 1787828064] req@ffff9e35da6f7840 x1874672085521152/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828089 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828064.776915:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7840 x1874672085521152/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828089 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828064.776925:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828064.776931:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828064.776936:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085521152:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828064.776945:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085521280:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828064.776967:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828064.776983:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828064/real 1787828064] req@ffff9e35da6f4780 x1874672085521280/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828089 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828064.776997:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4780 x1874672085521280/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828089 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828064.777007:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828064.777010:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828064.777014:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085521280:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828064.777022:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085521408:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828064.777044:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828064.777061:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828064/real 1787828064] req@ffff9e35da6f43c0 x1874672085521408/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828089 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828064.777074:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f43c0 x1874672085521408/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828089 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828064.777084:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828064.777087:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828064.777090:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085521408:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828064.777096:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085521536:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828064.777133:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828064.777149:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828064/real 1787828064] req@ffff9e35da6f4b40 x1874672085521536/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828089 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828064.777167:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4b40 x1874672085521536/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828089 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828064.777180:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828064.777185:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828064.777191:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085521536:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828064.777201:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085521664:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828064.777226:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828064.777255:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828064/real 1787828064] req@ffff9e35da6f61c0 x1874672085521664/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828089 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828064.777268:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f61c0 x1874672085521664/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828089 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828064.777277:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828064.777280:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828064.777284:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085521664:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828064.777290:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085521792:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828064.777307:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828064.777316:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828064/real 1787828064] req@ffff9e35da6f5a40 x1874672085521792/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828089 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828064.777328:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5a40 x1874672085521792/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828089 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828064.777336:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828064.777339:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828064.777342:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085521792:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828064.777347:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085521920:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828064.777361:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828064.777370:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828064/real 1787828064] req@ffff9e35da6f4000 x1874672085521920/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828089 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828064.777381:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4000 x1874672085521920/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828089 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828064.777389:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828064.777392:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828064.777395:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085521920:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787828064.903938:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828064.904051:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828065.927992:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828065.928014:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828066.951971:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828066.952010:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828067.976024:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828067.976181:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828068.999947:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828069.000025:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828069.896272:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085522048:192.168.203.141@tcp:250:kworker.0 00000800:00000100:2.0:1787828069.896373:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828069.896438:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828069/real 1787828069] req@ffff9e35da718780 x1874672085522048/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828094 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828069.896492:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718780 x1874672085522048/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828094 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828069.896520:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828069.896552:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828069.896574:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085522048:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1787828069.896622:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085522176:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828069.896662:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828069.896685:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828069/real 1787828069] req@ffff9e35da71a580 x1874672085522176/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828094 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828069.896760:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672085522176/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828094 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828069.896774:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828069.896786:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828069.896797:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085522176:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828069.896805:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085522304:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828069.896889:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828069.896909:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828069/real 1787828069] req@ffff9e35da71a940 x1874672085522304/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828094 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828069.896939:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672085522304/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828094 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828069.896951:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828069.896954:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828069.896956:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085522304:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828069.896978:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085522432:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828069.897002:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828069.897014:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828069/real 1787828069] req@ffff9e35da7183c0 x1874672085522432/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828094 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828069.897036:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672085522432/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828094 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828069.897062:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828069.897081:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828069.897101:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085522432:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828069.897116:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085522560:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828069.897146:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828069.897168:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828069/real 1787828069] req@ffff9e35da718b40 x1874672085522560/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828094 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828069.897181:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718b40 x1874672085522560/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828094 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828069.897195:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828069.897198:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828069.897200:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085522560:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828069.897220:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085522688:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828069.897235:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828069.897242:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828069/real 1787828069] req@ffff9e35da718f00 x1874672085522688/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828094 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828069.897251:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672085522688/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828094 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828069.897258:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828069.897260:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828069.897262:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085522688:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828069.897267:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085522816:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828069.897279:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828069.897286:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828069/real 1787828069] req@ffff9e35da719a40 x1874672085522816/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828094 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828069.897295:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672085522816/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828094 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828069.897301:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828069.897303:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828069.897306:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085522816:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828069.897310:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085522944:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828069.897321:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828069.897339:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828069/real 1787828069] req@ffff9e35da71a1c0 x1874672085522944/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828094 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828069.897347:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a1c0 x1874672085522944/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828094 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828069.897354:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828069.897356:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828069.897358:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085522944:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828069.897363:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085523072:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828069.897381:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828069.897388:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828069/real 1787828069] req@ffff9e35da718000 x1874672085523072/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828094 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828069.897397:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718000 x1874672085523072/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828094 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828069.897404:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828069.897406:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828069.897409:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085523072:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828070.024032:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828070.024116:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787828071.047992:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828072.071934:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828072.072017:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787828073.095974:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828073.095979:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828074.119863:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828074.119931:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828075.015914:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828075.015923:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828075.015927:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828075.015937:0:27606:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1510 00000100:00080000:2.0:1787828075.015958:0:27606:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828075.015966:0:27606:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787828075.016009:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e1c0 x1874672085523200/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828075.016041:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828075.016045:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828075.016047:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828075.016051:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1510 00000100:00080000:2.0:1787828075.016061:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828075.016065:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828075.016075:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828075.016090:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f480 x1874672085523328/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828075.016101:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828075.016105:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787828075.016107:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828075.016110:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1510 00000100:00080000:2.0:1787828075.016113:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828075.016116:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828075.016121:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828075.016130:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f0c0 x1874672085523456/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828075.016147:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828075.016150:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787828075.016152:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828075.016154:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1510 00000100:00080000:2.0:1787828075.016157:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828075.016163:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828075.016170:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:2.0:1787828075.016182:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f840 x1874672085523584/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828075.016190:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828075.016193:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787828075.016194:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828075.016198:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1510 00000100:00080000:2.0:1787828075.016200:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828075.016212:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828075.016216:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:2.0:1787828075.016228:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cb40 x1874672085523712/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828075.016239:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828075.016249:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828075.016254:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828075.016262:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1510 00000100:00080000:2.0:1787828075.016266:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828075.016269:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828075.016273:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828075.016283:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c000 x1874672085523840/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828075.016309:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828075.016317:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787828075.016319:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828075.016323:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1510 00000100:00080000:2.0:1787828075.016328:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828075.016333:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828075.016339:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828075.016354:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e580 x1874672085523968/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828075.016368:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828075.016371:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787828075.016373:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828075.016376:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1510 00000100:00080000:2.0:1787828075.016379:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828075.016381:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828075.016385:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:2.0:1787828075.016401:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cf00 x1874672085524096/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828075.016416:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828075.016423:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787828075.016427:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828075.016437:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1510 00000100:00080000:2.0:1787828075.016443:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828075.016455:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828075.016465:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:2.0:1787828075.016495:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498fc00 x1874672085524224/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828075.143950:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828075.144079:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828076.167898:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828076.168044:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828077.191920:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828077.192054:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828078.215877:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828078.216020:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828079.239984:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828079.240120:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828080.263939:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828080.263980:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828081.287959:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828082.311937:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787828082.311973:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828083.336046:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828083.336108:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828084.359960:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00080000:2.0:1787828085.255878:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828085.255886:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828085.255889:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828085.255897:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1520 00000100:00080000:2.0:1787828085.255906:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828085.255912:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787828085.255942:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498de00 x1874672085525504/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828085.255965:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828085.255968:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828085.255970:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828085.255972:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1520 00000100:00080000:2.0:1787828085.255974:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828085.255977:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828085.255984:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828085.255993:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498ed00 x1874672085525632/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828085.256002:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828085.256004:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787828085.256006:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828085.256011:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1520 00000100:00080000:2.0:1787828085.256017:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828085.256021:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828085.256024:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828085.256033:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c3c0 x1874672085525760/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828085.256039:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828085.256042:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787828085.256044:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828085.256047:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1520 00000100:00080000:2.0:1787828085.256050:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828085.256052:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828085.256055:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:2.0:1787828085.256062:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e940 x1874672085525888/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828085.256068:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828085.256073:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787828085.256076:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828085.256080:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1520 00000100:00080000:2.0:1787828085.256083:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828085.256087:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828085.256091:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:2.0:1787828085.256105:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498d680 x1874672085526016/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828085.256131:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828085.256138:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828085.256144:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828085.256154:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1520 00000100:00080000:2.0:1787828085.256160:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828085.256168:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828085.256174:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828085.256185:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c780 x1874672085526144/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828085.256199:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828085.256203:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787828085.256204:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828085.256208:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1520 00000100:00080000:2.0:1787828085.256212:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828085.256216:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828085.256223:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828085.256236:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498de00 x1874672085526272/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828085.256261:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828085.256265:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787828085.256269:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828085.256277:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1520 00000100:00080000:2.0:1787828085.256281:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828085.256287:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828085.256293:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:2.0:1787828085.256302:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498fc00 x1874672085526400/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828085.256315:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828085.256321:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787828085.256323:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828085.256327:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1520 00000100:00080000:2.0:1787828085.256330:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828085.256334:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828085.256340:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:2.0:1787828085.256353:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cf00 x1874672085526528/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828085.383893:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828085.383919:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828085.383957:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828086.407959:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828086.408025:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828087.432014:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828088.456045:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828088.456097:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828089.479999:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828089.480208:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828089.480537:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828090.376145:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085526656:192.168.203.141@tcp:250:kworker.0 00000800:00000100:2.0:1787828090.376260:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828090.376322:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828090/real 1787828090] req@ffff9e35da7192c0 x1874672085526656/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828115 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828090.376376:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7192c0 x1874672085526656/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828115 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828090.376418:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828090.376437:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828090.376449:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085526656:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1787828090.376498:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085526784:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828090.376568:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828090.376603:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828090/real 1787828090] req@ffff9e35da718000 x1874672085526784/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828115 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828090.376680:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718000 x1874672085526784/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828115 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828090.376720:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828090.376733:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828090.376746:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085526784:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828090.376767:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085526912:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828090.376850:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828090.376874:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828090/real 1787828090] req@ffff9e35da71a1c0 x1874672085526912/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828115 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828090.376896:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a1c0 x1874672085526912/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828115 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828090.376913:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828090.376921:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828090.376929:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085526912:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828090.376943:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085527040:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828090.376985:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828090.377011:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828090/real 1787828090] req@ffff9e35da719a40 x1874672085527040/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828115 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828090.377036:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672085527040/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828115 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828090.377048:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828090.377051:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828090.377055:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085527040:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828090.377063:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085527168:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828090.377090:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828090.377103:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828090/real 1787828090] req@ffff9e35da718f00 x1874672085527168/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828115 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828090.377117:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672085527168/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828115 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828090.377132:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828090.377137:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828090.377143:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085527168:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828090.377153:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085527296:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828090.377188:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828090.377208:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828090/real 1787828090] req@ffff9e35da718b40 x1874672085527296/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828115 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828090.377231:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718b40 x1874672085527296/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828115 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828090.377241:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828090.377244:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828090.377249:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085527296:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828090.377275:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085527424:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828090.377300:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828090.377313:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828090/real 1787828090] req@ffff9e35da7183c0 x1874672085527424/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828115 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828090.377326:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672085527424/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828115 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828090.377338:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828090.377343:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828090.377349:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085527424:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828090.377359:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085527552:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828090.377393:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828090.377413:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828090/real 1787828090] req@ffff9e35da71a940 x1874672085527552/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828115 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828090.377436:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672085527552/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828115 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828090.377447:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828090.377451:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828090.377455:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085527552:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828090.377463:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085527680:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828090.377487:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828090.377500:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828090/real 1787828090] req@ffff9e35da71a580 x1874672085527680/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828115 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828090.377514:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672085527680/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828115 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828090.377523:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828090.377527:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828090.377530:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085527680:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787828090.503981:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:3.0:1787828091.527914:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828091.528017:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828092.551882:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828092.551929:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828093.575962:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828093.575999:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828094.599957:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828094.600029:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828095.495984:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828095.496001:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828095.496008:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828095.496024:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1530 00000100:00080000:2.0:1787828095.496041:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828095.496054:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787828095.496110:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e580 x1874672085527808/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828095.496155:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828095.496161:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828095.496163:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828095.496169:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1530 00000100:00080000:2.0:1787828095.496174:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828095.496180:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828095.496193:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828095.496211:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c000 x1874672085527936/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828095.496227:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828095.496235:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787828095.496240:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828095.496252:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1530 00000100:00080000:2.0:1787828095.496260:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828095.496264:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828095.496270:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828095.496286:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cb40 x1874672085528064/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828095.496299:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828095.496304:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787828095.496307:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828095.496311:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1530 00000100:00080000:2.0:1787828095.496317:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828095.496321:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828095.496327:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:2.0:1787828095.496343:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f840 x1874672085528192/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828095.496363:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828095.496370:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787828095.496373:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828095.496382:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1530 00000100:00080000:2.0:1787828095.496388:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828095.496392:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828095.496399:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:2.0:1787828095.496414:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f0c0 x1874672085528320/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828095.496427:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828095.496432:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828095.496435:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828095.496443:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1530 00000100:00080000:2.0:1787828095.496448:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828095.496461:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828095.496474:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828095.496496:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f480 x1874672085528448/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828095.496553:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828095.496565:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787828095.496577:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828095.496597:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1530 00000100:00080000:2.0:1787828095.496609:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828095.496618:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828095.496631:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828095.496671:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e1c0 x1874672085528576/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828095.496698:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828095.496705:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787828095.496710:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828095.496726:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1530 00000100:00080000:2.0:1787828095.496737:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828095.496746:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828095.496761:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:2.0:1787828095.496867:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e580 x1874672085528704/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828095.496897:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828095.496912:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787828095.496918:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828095.496925:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1530 00000100:00080000:2.0:1787828095.496933:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828095.496940:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828095.496949:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:2.0:1787828095.496981:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cf00 x1874672085528832/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787828095.623911:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828095.623991:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828096.647942:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828096.648050:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828097.671927:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828097.672021:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787828098.695989:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828099.719958:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828099.720089:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828100.616221:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085528960:192.168.203.141@tcp:250:kworker.0 00000800:00000100:2.0:1787828100.616318:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828100.616412:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828100/real 1787828100] req@ffff9e35da718780 x1874672085528960/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828125 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828100.616465:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718780 x1874672085528960/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828125 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828100.616515:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828100.616538:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828100.616554:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085528960:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1787828100.616595:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085529088:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828100.616651:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828100.616678:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828100/real 1787828100] req@ffff9e35da71a580 x1874672085529088/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828125 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828100.616717:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672085529088/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828125 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828100.616753:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828100.616761:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828100.616771:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085529088:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828100.616838:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085529216:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828100.616890:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828100.616923:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828100/real 1787828100] req@ffff9e35da71a940 x1874672085529216/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828125 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828100.616955:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672085529216/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828125 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828100.617012:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828100.617024:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828100.617041:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085529216:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828100.617063:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085529344:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828100.617113:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828100.617133:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828100/real 1787828100] req@ffff9e35da7183c0 x1874672085529344/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828125 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828100.617150:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672085529344/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828125 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828100.617161:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828100.617165:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828100.617169:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085529344:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828100.617190:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085529472:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828100.617211:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828100.617223:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828100/real 1787828100] req@ffff9e35da718b40 x1874672085529472/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828125 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828100.617237:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718b40 x1874672085529472/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828125 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828100.617248:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828100.617251:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828100.617256:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085529472:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828100.617263:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085529600:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828100.617280:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828100.617291:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828100/real 1787828100] req@ffff9e35da718f00 x1874672085529600/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828125 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828100.617308:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672085529600/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828125 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828100.617325:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828100.617330:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828100.617338:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085529600:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828100.617349:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085529728:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828100.617395:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828100.617418:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828100/real 1787828100] req@ffff9e35da719a40 x1874672085529728/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828125 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828100.617443:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672085529728/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828125 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828100.617453:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828100.617457:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828100.617461:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085529728:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828100.617469:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085529856:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828100.617526:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828100.617540:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828100/real 1787828100] req@ffff9e35da71a1c0 x1874672085529856/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828125 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828100.617555:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a1c0 x1874672085529856/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828125 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828100.617566:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828100.617570:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828100.617574:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085529856:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828100.617580:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085529984:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828100.617599:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828100.617610:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828100/real 1787828100] req@ffff9e35da718000 x1874672085529984/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828125 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828100.617624:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718000 x1874672085529984/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828125 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828100.617634:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828100.617638:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828100.617641:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085529984:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828100.743966:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828101.767929:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828101.768011:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828101.768108:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828102.792017:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828103.816006:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828104.840072:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828104.840073:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00080000:3.0:1787828105.736008:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828105.736026:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787828105.736031:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828105.736044:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1541 00000100:00080000:3.0:1787828105.736057:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828105.736066:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787828105.736125:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498f00 x1874672085530112/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828105.736185:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828105.736199:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828105.736201:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828105.736208:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1541 00000100:00080000:3.0:1787828105.736214:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828105.736220:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828105.736243:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828105.736259:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085530112:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1787828105.736268:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b840 x1874672085530240/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828105.736294:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828105.736299:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787828105.736302:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828105.736307:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1541 00000100:00080000:3.0:1787828105.736313:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828105.736318:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828105.736324:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828105.736339:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649bc00 x1874672085530368/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828105.736352:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828105.736357:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787828105.736361:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1787828105.736367:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1787828105.736372:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1541 00000100:00080000:3.0:1787828105.736380:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828105.736386:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828105.736393:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:3.0:1787828105.736408:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a940 x1874672085530496/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828105.736420:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828105.736426:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787828105.736430:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787828105.736435:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828105/real 1787828105] req@ffff9e35c6498f00 x1874672085530112/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828130 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828105.736468:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1541 00000100:00080000:3.0:1787828105.736484:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828105.736494:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787828105.736500:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498f00 x1874672085530112/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828130 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1787828105.736509:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00080000:2.0:1787828105.736522:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828105.736529:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828105.736546:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085530112:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1787828105.736548:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498b40 x1874672085530624/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828105.736579:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828105.736584:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828105.736587:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787828105.736593:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085530240:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1787828105.736601:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1541 00000100:00080000:3.0:1787828105.736610:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828105.736615:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828105.736624:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1787828105.736642:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828105.736648:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498780 x1874672085530752/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828105.736665:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828105.736670:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787828105.736674:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828105.736679:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1541 00000100:00080000:3.0:1787828105.736683:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828105.736687:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787828105.736694:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828105/real 1787828105] req@ffff9e35c649b840 x1874672085530240/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828130 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1787828105.736696:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828105.736711:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499680 x1874672085530880/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828105.736766:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828105.736771:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787828105.736773:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787828105.736782:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b840 x1874672085530240/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828130 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828105.736785:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1541 00000100:00080000:2.0:1787828105.736852:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828105.736855:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828105.736863:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:3.0:1787828105.736870:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787828105.736879:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085530240:192.168.203.141@tcp:38:kworker.0 00000008:00100000:3.0:1787828105.736883:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:2.0:1787828105.736894:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085530368:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787828105.736913:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a580 x1874672085531008/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828105.736934:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1787828105.736938:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1787828105.736950:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787828105.736956:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787828105.736965:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828105/real 1787828105] req@ffff9e35c649bc00 x1874672085530368/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828130 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828105.736971:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1541 00000100:00080000:3.0:1787828105.736987:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828105.737002:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828105.737014:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:2.0:1787828105.737018:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649bc00 x1874672085530368/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828130 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828105.737033:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1787828105.737041:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499a40 x1874672085531136/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828105.737042:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828105.737055:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085530368:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828105.737080:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085530496:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828105.737109:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828105.737126:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828105/real 1787828105] req@ffff9e35c649a940 x1874672085530496/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828130 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828105.737141:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a940 x1874672085530496/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828130 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828105.737151:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828105.737155:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828105.737159:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085530496:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828105.737166:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085530624:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828105.737184:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828105.737195:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828105/real 1787828105] req@ffff9e35c6498b40 x1874672085530624/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828130 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828105.737209:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498b40 x1874672085530624/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828130 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828105.737228:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828105.737232:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828105.737235:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085530624:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828105.737261:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085530752:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828105.737281:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828105.737293:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828105/real 1787828105] req@ffff9e35c6498780 x1874672085530752/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828130 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828105.737306:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498780 x1874672085530752/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828130 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828105.737316:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828105.737320:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828105.737323:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085530752:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828105.737330:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085530880:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828105.737348:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828105.737359:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828105/real 1787828105] req@ffff9e35c6499680 x1874672085530880/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828130 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828105.737374:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499680 x1874672085530880/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828130 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828105.737383:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828105.737386:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828105.737391:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085530880:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828105.737397:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085531008:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828105.737414:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828105.737425:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828105/real 1787828105] req@ffff9e35c649a580 x1874672085531008/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828130 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828105.737466:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a580 x1874672085531008/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828130 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828105.737478:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828105.737482:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828105.737487:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085531008:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828105.737494:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085531136:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828105.737520:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828105.737533:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828105/real 1787828105] req@ffff9e35c6499a40 x1874672085531136/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828130 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828105.737548:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499a40 x1874672085531136/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828130 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828105.737558:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828105.737562:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828105.737566:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085531136:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828105.863999:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828106.887958:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828107.912024:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828108.936052:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828109.959994:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00080000:2.0:1787828110.856041:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828110.856053:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828110.856058:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828110.856069:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1546 00000100:00080000:2.0:1787828110.856082:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828110.856091:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787828110.856134:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498fc00 x1874672085531264/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828110.856173:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828110.856178:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828110.856180:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828110.856185:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1546 00000100:00080000:2.0:1787828110.856189:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828110.856193:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828110.856205:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828110.856220:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498de00 x1874672085531392/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828110.856233:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828110.856237:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787828110.856240:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828110.856245:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1546 00000100:00080000:2.0:1787828110.856248:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828110.856252:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828110.856264:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828110.856285:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c780 x1874672085531520/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828110.856298:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828110.856302:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787828110.856305:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828110.856310:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1546 00000100:00080000:2.0:1787828110.856314:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828110.856319:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828110.856341:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:2.0:1787828110.856372:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498d680 x1874672085531648/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828110.856394:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828110.856423:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787828110.856428:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828110.856453:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1546 00000100:00080000:2.0:1787828110.856469:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828110.856480:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828110.856493:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:2.0:1787828110.856536:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e940 x1874672085531776/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828110.856562:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828110.856572:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828110.856575:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828110.856583:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1546 00000100:00080000:2.0:1787828110.856591:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828110.856597:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828110.856604:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828110.856628:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c3c0 x1874672085531904/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828110.856708:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828110.856723:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787828110.856728:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828110.856740:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1546 00000100:00080000:2.0:1787828110.856750:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828110.856762:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828110.856777:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828110.856855:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498ed00 x1874672085532032/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828110.856875:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828110.856880:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787828110.856884:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828110.856898:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1546 00000100:00080000:2.0:1787828110.856907:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828110.856913:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828110.856920:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:2.0:1787828110.856942:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498de00 x1874672085532160/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828110.856993:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828110.857002:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787828110.857005:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828110.857026:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1546 00000100:00080000:2.0:1787828110.857036:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828110.857047:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828110.857055:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:2.0:1787828110.857078:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498fc00 x1874672085532288/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828110.983930:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828110.984010:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828112.007967:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828113.031961:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828114.055967:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828114.056006:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828115.079899:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787828115.079997:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828116.104048:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828117.127958:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828117.128052:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828118.152030:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787828118.152117:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787828119.176038:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787828120.199976:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828120.200028:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828121.096223:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085533568:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1787828121.096385:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085533696:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828121.096469:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085533824:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828121.096518:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085533952:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828121.096563:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085534080:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828121.096590:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085534208:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828121.096636:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085534336:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828121.096662:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085534464:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828121.096697:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085534592:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787828121.223939:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787828121.224155:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828122.248033:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828122.248070:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828123.271962:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000400:00000100:3.0:1787828124.168033:0:2399:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.141:988 00000400:00000100:3.0:1787828124.168072:0:2399:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.141@tcp at host 192.168.203.141 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1787828124.168086:0:2399:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1787828124.168146:0:2399:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000800:00000100:3.0:1787828124.168161:0:2399:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1787828124.168171:0:2399:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1787828124.168178:0:2399:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1787828124.168186:0:2399:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:2.0:1787828124.168189:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828121/real 1787828124] req@ffff9e35da71a580 x1874672085534592/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828146 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:3.0:1787828124.168209:0:2399:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1787828124.168221:0:2399:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1787828124.168231:0:2399:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:2.0:1787828124.168248:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672085534592/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828146 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828124.168263:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828124.168270:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828124.168275:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085534592:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828124.168283:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828121/real 1787828124] req@ffff9e35da7183c0 x1874672085534336/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828146 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828124.168297:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672085534336/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828146 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828124.168307:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828124.168322:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828124.168326:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085534336:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828124.168333:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828121/real 1787828124] req@ffff9e35da71a940 x1874672085534464/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828146 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828124.168347:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672085534464/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828146 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828124.168357:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828124.168361:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828124.168365:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085534464:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828124.168372:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828121/real 1787828124] req@ffff9e35da71a1c0 x1874672085533824/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828146 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828124.168386:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a1c0 x1874672085533824/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828146 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828124.168392:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828124.168395:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828124.168397:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085533824:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828124.168403:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828121/real 1787828124] req@ffff9e35da719a40 x1874672085533952/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828146 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828124.168421:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672085533952/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828146 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828124.168430:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828124.168434:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828124.168438:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085533952:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828124.168446:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828121/real 1787828124] req@ffff9e35da718f00 x1874672085534080/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828146 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828124.168457:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672085534080/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828146 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828124.168470:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828124.168474:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828124.168478:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085534080:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828124.168484:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828121/real 1787828124] req@ffff9e35da718b40 x1874672085534208/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828146 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828124.168495:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718b40 x1874672085534208/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828146 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828124.168508:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828124.168511:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828124.168515:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085534208:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828124.168521:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828121/real 1787828124] req@ffff9e35da7192c0 x1874672085533568/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828146 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828124.168538:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7192c0 x1874672085533568/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828146 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828124.168545:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828124.168549:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828124.168553:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085533568:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1787828124.168559:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828121/real 1787828124] req@ffff9e35da718000 x1874672085533696/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828146 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828124.168572:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718000 x1874672085533696/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828146 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828124.168581:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828124.168585:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828124.168589:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085533696:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1787828124.295955:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828125.319967:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787828125.320007:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828126.215959:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828126.215972:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828126.215978:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828126.215991:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1561 00000100:00080000:2.0:1787828126.216005:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828126.216015:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787828126.216062:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cf00 x1874672085534720/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828126.216105:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828126.216111:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828126.216113:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828126.216118:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1561 00000100:00080000:2.0:1787828126.216124:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828126.216129:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828126.216141:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828126.216160:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e580 x1874672085534848/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828126.216187:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828126.216194:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787828126.216199:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828126.216207:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1561 00000100:00080000:2.0:1787828126.216237:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828126.216260:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828126.216280:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828126.216340:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e1c0 x1874672085534976/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828126.216374:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828126.216381:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787828126.216386:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828126.216394:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1561 00000100:00080000:2.0:1787828126.216402:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828126.216417:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828126.216431:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:2.0:1787828126.216457:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f480 x1874672085535104/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828126.216495:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828126.216513:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787828126.216522:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828126.216541:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1561 00000100:00080000:2.0:1787828126.216556:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828126.216569:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828126.216580:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:2.0:1787828126.216611:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f0c0 x1874672085535232/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828126.216647:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828126.216656:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828126.216666:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828126.216686:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1561 00000100:00080000:2.0:1787828126.216698:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828126.216717:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828126.216730:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828126.216770:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f840 x1874672085535360/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828126.216857:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828126.216868:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787828126.216877:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828126.216888:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1561 00000100:00080000:2.0:1787828126.216900:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828126.216917:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828126.216936:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828126.216969:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cb40 x1874672085535488/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828126.217001:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828126.217013:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787828126.217023:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828126.217038:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1561 00000100:00080000:2.0:1787828126.217050:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828126.217060:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828126.217071:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:2.0:1787828126.217135:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c000 x1874672085535616/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828126.217160:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828126.217173:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787828126.217180:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828126.217192:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1561 00000100:00080000:2.0:1787828126.217201:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828126.217217:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828126.217236:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:2.0:1787828126.217274:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e580 x1874672085535744/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828126.344035:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828126.344117:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828127.367963:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787828127.368065:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828128.391959:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787828128.392018:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787828129.416004:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828129.416018:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828130.439934:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828130.440003:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828131.463959:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828132.487900:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828132.487986:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828133.511945:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828134.535903:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787828134.535940:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828135.559856:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828135.559921:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828136.455937:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828136.455945:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828136.455949:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828136.455957:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1571 00000100:00080000:2.0:1787828136.455966:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828136.455971:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787828136.456017:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cf00 x1874672085537024/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828136.456035:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828136.456039:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828136.456041:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828136.456045:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1571 00000100:00080000:2.0:1787828136.456050:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828136.456054:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828136.456062:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828136.456075:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498fc00 x1874672085537152/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828136.456085:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828136.456089:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787828136.456091:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828136.456097:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1571 00000100:00080000:2.0:1787828136.456103:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828136.456106:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828136.456111:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828136.456125:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498de00 x1874672085537280/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828136.456137:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828136.456141:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787828136.456144:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828136.456152:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1571 00000100:00080000:2.0:1787828136.456161:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828136.456165:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828136.456172:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:2.0:1787828136.456183:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498ed00 x1874672085537408/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828136.456202:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828136.456207:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787828136.456209:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828136.456215:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1571 00000100:00080000:2.0:1787828136.456220:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828136.456240:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828136.456249:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:2.0:1787828136.456267:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c3c0 x1874672085537536/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828136.456285:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828136.456288:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828136.456290:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828136.456295:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1571 00000100:00080000:2.0:1787828136.456298:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828136.456301:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828136.456305:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828136.456317:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e940 x1874672085537664/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828136.456325:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828136.456327:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787828136.456329:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828136.456331:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1571 00000100:00080000:2.0:1787828136.456334:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828136.456337:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828136.456344:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828136.456356:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498d680 x1874672085537792/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828136.456369:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828136.456376:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787828136.456380:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828136.456386:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1571 00000100:00080000:2.0:1787828136.456389:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828136.456394:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828136.456402:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:2.0:1787828136.456414:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c780 x1874672085537920/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828136.456425:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828136.456429:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787828136.456431:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828136.456441:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1571 00000100:00080000:2.0:1787828136.456449:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828136.456458:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828136.456464:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:2.0:1787828136.456478:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cf00 x1874672085538048/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828136.583974:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828136.584046:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828137.607920:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:3.0:1787828137.607943:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787828137.608132:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828138.631998:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828138.632186:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828139.655992:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828139.656032:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828140.679952:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828140.680004:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828141.576135:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085538176:192.168.203.141@tcp:250:kworker.0 00000800:00000100:2.0:1787828141.576227:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828141.576278:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828141/real 1787828141] req@ffff9e35da718780 x1874672085538176/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828166 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828141.576321:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718780 x1874672085538176/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828166 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828141.576339:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828141.576346:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828141.576353:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085538176:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1787828141.576383:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085538304:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828141.576413:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828141.576428:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828141/real 1787828141] req@ffff9e35da718000 x1874672085538304/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828166 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828141.576451:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718000 x1874672085538304/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828166 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828141.576459:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828141.576462:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828141.576466:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085538304:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828141.576473:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085538432:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828141.576485:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828141.576492:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828141/real 1787828141] req@ffff9e35da7192c0 x1874672085538432/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828166 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828141.576500:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7192c0 x1874672085538432/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828166 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828141.576505:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828141.576506:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828141.576508:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085538432:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828141.576512:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085538560:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828141.576523:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828141.576528:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828141/real 1787828141] req@ffff9e35da718b40 x1874672085538560/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828166 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828141.576536:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718b40 x1874672085538560/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828166 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828141.576541:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828141.576543:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828141.576545:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085538560:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828141.576548:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085538688:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828141.576558:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828141.576564:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828141/real 1787828141] req@ffff9e35da718f00 x1874672085538688/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828166 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828141.576572:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672085538688/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828166 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828141.576578:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828141.576579:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828141.576581:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085538688:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828141.576585:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085538816:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828141.576595:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828141.576601:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828141/real 1787828141] req@ffff9e35da719a40 x1874672085538816/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828166 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828141.576609:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672085538816/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828166 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828141.576616:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828141.576618:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828141.576621:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085538816:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828141.576628:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085538944:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828141.576646:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828141.576658:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828141/real 1787828141] req@ffff9e35da71a1c0 x1874672085538944/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828166 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828141.576671:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a1c0 x1874672085538944/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828166 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828141.576683:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828141.576685:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828141.576688:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085538944:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828141.576707:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085539072:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828141.576721:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828141.576728:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828141/real 1787828141] req@ffff9e35da71a940 x1874672085539072/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828166 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828141.576736:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672085539072/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828166 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828141.576742:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828141.576744:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828141.576746:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085539072:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828141.576750:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085539200:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828141.576759:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828141.576765:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828141/real 1787828141] req@ffff9e35da7183c0 x1874672085539200/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828166 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828141.576772:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672085539200/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828166 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828141.576777:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828141.576779:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828141.576782:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085539200:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828141.703899:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787828141.703947:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828142.727916:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787828142.728032:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828142.728124:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828143.751960:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828143.751962:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828144.776079:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787828144.776330:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828145.799924:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828145.799994:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828146.695902:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828146.695911:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787828146.695916:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828146.695927:0:27606:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1582 00000100:00080000:3.0:1787828146.695938:0:27606:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828146.695945:0:27606:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787828146.695982:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c64992c0 x1874672085539328/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828146.696012:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828146.696016:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828146.696018:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828146.696022:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1582 00000100:00080000:3.0:1787828146.696024:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828146.696028:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828146.696037:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828146.696047:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498000 x1874672085539456/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828146.696055:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828146.696058:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787828146.696059:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828146.696062:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1582 00000100:00080000:3.0:1787828146.696064:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828146.696066:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828146.696069:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828146.696076:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a1c0 x1874672085539584/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828146.696083:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828146.696086:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787828146.696088:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828146.696091:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1582 00000100:00080000:3.0:1787828146.696093:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828146.696097:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828146.696102:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:3.0:1787828146.696110:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c64983c0 x1874672085539712/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828146.696116:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c64992c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085539328:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1787828146.696123:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828146.696127:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787828146.696129:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828146.696133:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1582 00000100:00080000:3.0:1787828146.696135:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828146.696138:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828146.696142:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:3.0:1787828146.696167:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499e00 x1874672085539840/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828146.696178:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828146.696181:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828146.696183:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828146.696187:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1582 00000100:00080000:3.0:1787828146.696191:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828146.696211:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828146.696217:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1787828146.696220:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828146.696236:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b480 x1874672085539968/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828146.696247:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828146.696250:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787828146.696253:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828146.696259:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1582 00000100:00080000:3.0:1787828146.696264:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828146.696268:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787828146.696274:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828146/real 1787828146] req@ffff9e35c64992c0 x1874672085539328/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828171 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1787828146.696276:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828146.696295:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b0c0 x1874672085540096/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828146.696306:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c64992c0 x1874672085539328/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828171 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828146.696309:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828146.696322:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787828146.696328:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828146.696338:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828146.696338:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1582 00000100:00080000:3.0:1787828146.696342:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828146.696349:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:3.0:1787828146.696354:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787828146.696361:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c64992c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085539328:192.168.203.141@tcp:250:kworker.0 00000008:00100000:3.0:1787828146.696363:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:2.0:1787828146.696376:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085539456:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787828146.696387:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649ad00 x1874672085540224/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828146.696404:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1787828146.696409:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1787828146.696416:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787828146.696422:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787828146.696430:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828146/real 1787828146] req@ffff9e35c6498000 x1874672085539456/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828171 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828146.696435:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1582 00000100:00080000:3.0:1787828146.696451:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787828146.696458:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498000 x1874672085539456/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828171 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828146.696464:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828146.696476:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00080000:2.0:1787828146.696479:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828146.696487:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828146.696491:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085539456:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787828146.696499:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499a40 x1874672085540352/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828146.696503:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085539584:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828146.696559:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828146.696571:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828146/real 1787828146] req@ffff9e35c649a1c0 x1874672085539584/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828171 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828146.696581:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a1c0 x1874672085539584/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828171 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828146.696589:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828146.696592:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828146.696596:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085539584:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828146.696622:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c64983c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085539712:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828146.696636:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828146.696644:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828146/real 1787828146] req@ffff9e35c64983c0 x1874672085539712/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828171 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828146.696653:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c64983c0 x1874672085539712/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828171 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828146.696659:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828146.696662:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828146.696664:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c64983c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085539712:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828146.696669:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085539840:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828146.696680:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828146.696687:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828146/real 1787828146] req@ffff9e35c6499e00 x1874672085539840/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828171 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828146.696707:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499e00 x1874672085539840/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828171 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828146.696717:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828146.696720:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828146.696723:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085539840:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828146.696728:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085539968:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828146.696742:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828146.696750:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828146/real 1787828146] req@ffff9e35c649b480 x1874672085539968/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828171 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828146.696759:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b480 x1874672085539968/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828171 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828146.696765:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828146.696767:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828146.696770:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085539968:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828146.696774:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085540096:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828146.696786:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828146.696793:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828146/real 1787828146] req@ffff9e35c649b0c0 x1874672085540096/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828171 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828146.696873:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b0c0 x1874672085540096/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828171 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828146.696881:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828146.696884:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828146.696887:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085540096:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828146.696893:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085540224:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828146.696914:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828146.696925:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828146/real 1787828146] req@ffff9e35c649ad00 x1874672085540224/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828171 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828146.696936:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649ad00 x1874672085540224/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828171 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828146.696944:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828146.696947:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828146.696950:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085540224:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828146.696956:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085540352:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828146.696970:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828146.696978:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828146/real 1787828146] req@ffff9e35c6499a40 x1874672085540352/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828171 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828146.696988:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499a40 x1874672085540352/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828171 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828146.697002:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828146.697005:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828146.697008:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085540352:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828146.823898:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828147.847947:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787828147.848066:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828148.871919:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828148.872043:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828149.895985:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828149.896032:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828150.919955:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828150.920081:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828151.816296:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085540480:192.168.203.141@tcp:250:kworker.0 00000800:00000100:2.0:1787828151.816391:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828151.816443:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828151/real 1787828151] req@ffff9e35da71a580 x1874672085540480/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828176 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828151.816488:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672085540480/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828176 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828151.816520:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828151.816530:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828151.816538:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085540480:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1787828151.816581:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085540608:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828151.816617:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828151.816639:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828151/real 1787828151] req@ffff9e35da7183c0 x1874672085540608/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828176 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828151.816669:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672085540608/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828176 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828151.816706:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828151.816713:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828151.816724:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085540608:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828151.816736:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085540736:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828151.816769:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828151.816786:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828151/real 1787828151] req@ffff9e35da71a940 x1874672085540736/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828176 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828151.816863:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672085540736/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828176 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828151.816871:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828151.816875:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828151.816879:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085540736:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828151.816886:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085540864:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828151.816917:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828151.816935:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828151/real 1787828151] req@ffff9e35da71a1c0 x1874672085540864/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828176 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828151.816953:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a1c0 x1874672085540864/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828176 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828151.816975:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828151.816980:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828151.816985:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085540864:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828151.816994:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085540992:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828151.817023:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828151.817037:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828151/real 1787828151] req@ffff9e35da719a40 x1874672085540992/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828176 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828151.817066:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672085540992/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828176 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828151.817077:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828151.817080:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828151.817084:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085540992:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828151.817091:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085541120:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828151.817114:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828151.817178:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828151/real 1787828151] req@ffff9e35da718f00 x1874672085541120/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828176 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828151.817190:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672085541120/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828176 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828151.817198:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828151.817201:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828151.817205:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085541120:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828151.817227:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085541248:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828151.817247:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828151.817256:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828151/real 1787828151] req@ffff9e35da718b40 x1874672085541248/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828176 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828151.817267:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718b40 x1874672085541248/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828176 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828151.817275:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828151.817278:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828151.817281:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085541248:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828151.817286:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085541376:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828151.817299:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828151.817307:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828151/real 1787828151] req@ffff9e35da7192c0 x1874672085541376/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828176 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828151.817317:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7192c0 x1874672085541376/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828176 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828151.817325:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828151.817327:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828151.817330:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085541376:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828151.817335:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085541504:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828151.817348:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828151.817357:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828151/real 1787828151] req@ffff9e35da718000 x1874672085541504/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828176 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828151.817367:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718000 x1874672085541504/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828176 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828151.817375:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828151.817377:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828151.817380:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085541504:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828151.943986:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828151.944011:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828152.967983:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828152.968102:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828153.992037:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828153.992173:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828155.015945:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828155.016035:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828156.040053:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00080000:2.0:1787828156.935968:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828156.935977:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828156.935981:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828156.935989:0:27606:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1592 00000100:00080000:2.0:1787828156.935999:0:27606:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828156.936004:0:27606:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787828156.936036:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e580 x1874672085541632/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828156.936065:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828156.936068:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828156.936070:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828156.936073:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1592 00000100:00080000:2.0:1787828156.936075:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828156.936079:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828156.936100:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828156.936112:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c000 x1874672085541760/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828156.936122:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828156.936125:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787828156.936126:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828156.936130:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1592 00000100:00080000:2.0:1787828156.936135:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828156.936137:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828156.936141:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828156.936156:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cb40 x1874672085541888/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828156.936171:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828156.936192:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787828156.936196:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828156.936200:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1592 00000100:00080000:2.0:1787828156.936203:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828156.936205:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828156.936209:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:2.0:1787828156.936233:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f840 x1874672085542016/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828156.936244:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828156.936250:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787828156.936251:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828156.936259:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1592 00000100:00080000:2.0:1787828156.936266:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828156.936271:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828156.936276:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:2.0:1787828156.936287:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f0c0 x1874672085542144/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828156.936294:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828156.936296:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828156.936297:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828156.936300:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1592 00000100:00080000:2.0:1787828156.936304:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828156.936307:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828156.936310:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828156.936323:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f480 x1874672085542272/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828156.936333:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828156.936337:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787828156.936339:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828156.936343:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1592 00000100:00080000:2.0:1787828156.936347:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828156.936350:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828156.936356:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828156.936366:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e1c0 x1874672085542400/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828156.936374:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828156.936376:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787828156.936378:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828156.936381:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1592 00000100:00080000:2.0:1787828156.936384:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828156.936387:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828156.936392:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:2.0:1787828156.936404:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cf00 x1874672085542528/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828156.936414:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828156.936419:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787828156.936424:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828156.936439:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1592 00000100:00080000:2.0:1787828156.936446:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828156.936459:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828156.936477:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:2.0:1787828156.936491:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c780 x1874672085542656/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828157.063967:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828157.064101:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828158.087910:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828158.087949:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828159.112050:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828160.135915:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787828161.159968:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828161.160018:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828162.056285:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085542784:192.168.203.141@tcp:250:kworker.0 00000800:00000100:2.0:1787828162.056439:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828162.056539:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828162/real 1787828162] req@ffff9e35da718780 x1874672085542784/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828187 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828162.056570:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718780 x1874672085542784/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828187 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828162.056586:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828162.056591:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828162.056597:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085542784:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1787828162.056608:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085542912:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828162.056641:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828162.056657:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828162/real 1787828162] req@ffff9e35da718000 x1874672085542912/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828187 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828162.056673:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718000 x1874672085542912/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828187 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828162.056684:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828162.056689:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828162.056693:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085542912:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828162.056700:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085543040:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828162.056720:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828162.056732:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828162/real 1787828162] req@ffff9e35da7192c0 x1874672085543040/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828187 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828162.056747:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7192c0 x1874672085543040/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828187 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828162.056756:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828162.056760:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828162.056781:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085543040:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828162.056789:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085543168:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828162.056859:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828162.056892:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828162/real 1787828162] req@ffff9e35da718b40 x1874672085543168/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828187 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828162.056908:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718b40 x1874672085543168/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828187 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828162.056919:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828162.056923:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828162.056927:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085543168:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828162.056971:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085543296:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828162.056994:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828162.057006:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828162/real 1787828162] req@ffff9e35da718f00 x1874672085543296/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828187 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828162.057020:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672085543296/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828187 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828162.057032:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828162.057036:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828162.057040:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085543296:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828162.057102:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085543424:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828162.057127:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828162.057139:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828162/real 1787828162] req@ffff9e35da719a40 x1874672085543424/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828187 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828162.057153:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672085543424/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828187 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828162.057167:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828162.057172:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828162.057177:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085543424:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828162.057188:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085543552:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828162.057218:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828162.057255:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828162/real 1787828162] req@ffff9e35da71a1c0 x1874672085543552/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828187 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828162.057274:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a1c0 x1874672085543552/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828187 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828162.057285:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828162.057289:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828162.057294:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085543552:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828162.057302:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085543680:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828162.057328:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828162.057341:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828162/real 1787828162] req@ffff9e35da71a940 x1874672085543680/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828187 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828162.057355:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672085543680/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828187 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828162.057373:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828162.057379:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828162.057386:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085543680:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828162.057397:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085543808:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828162.057445:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828162.057464:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828162/real 1787828162] req@ffff9e35da7183c0 x1874672085543808/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828187 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828162.057480:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672085543808/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828187 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828162.057506:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828162.057511:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828162.057515:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085543808:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787828162.183960:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828162.183998:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:3.0:1787828163.207938:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828163.208014:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828163.208130:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828164.232117:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828164.232137:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828165.255995:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828165.256089:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828165.256125:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828166.279978:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828166.280092:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828167.176322:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085543936:192.168.203.141@tcp:250:kworker.0 00000800:00000100:2.0:1787828167.176462:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828167.176561:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828167/real 1787828167] req@ffff9e35da6f61c0 x1874672085543936/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828192 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828167.176629:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f61c0 x1874672085543936/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828192 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828167.176703:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828167.176724:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828167.176748:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085543936:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1787828167.176770:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085544064:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828167.176926:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828167.176973:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828167/real 1787828167] req@ffff9e35da6f7480 x1874672085544064/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828192 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828167.177005:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7480 x1874672085544064/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828192 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828167.177058:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828167.177081:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828167.177096:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085544064:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828167.177177:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085544192:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828167.177246:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828167.177267:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828167/real 1787828167] req@ffff9e35da6f6d00 x1874672085544192/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828192 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828167.177283:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6d00 x1874672085544192/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828192 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828167.177294:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828167.177299:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828167.177303:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085544192:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828167.177311:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085544320:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828167.177332:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828167.177343:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828167/real 1787828167] req@ffff9e35da6f7840 x1874672085544320/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828192 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828167.177357:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7840 x1874672085544320/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828192 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828167.177367:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828167.177371:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828167.177374:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085544320:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828167.177382:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085544448:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828167.177401:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828167.177411:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828167/real 1787828167] req@ffff9e35da6f4780 x1874672085544448/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828192 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828167.177426:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4780 x1874672085544448/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828192 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828167.177436:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828167.177439:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828167.177443:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085544448:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828167.177450:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085544576:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828167.177469:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828167.177479:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828167/real 1787828167] req@ffff9e35da6f43c0 x1874672085544576/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828192 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828167.177493:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f43c0 x1874672085544576/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828192 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828167.177502:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828167.177506:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828167.177509:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085544576:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828167.177516:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085544704:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828167.177534:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828167.177544:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828167/real 1787828167] req@ffff9e35da6f4b40 x1874672085544704/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828192 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828167.177557:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4b40 x1874672085544704/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828192 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828167.177575:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828167.177579:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828167.177582:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085544704:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828167.177589:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085544832:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828167.177621:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828167.177640:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828167/real 1787828167] req@ffff9e35da6f5a40 x1874672085544832/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828192 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828167.177663:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5a40 x1874672085544832/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828192 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828167.177681:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828167.177687:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828167.177694:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085544832:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828167.177727:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085544960:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828167.177755:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828167.177769:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828167/real 1787828167] req@ffff9e35da6f4000 x1874672085544960/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828192 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828167.177784:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4000 x1874672085544960/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828192 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828167.177794:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828167.177831:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828167.177837:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085544960:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828167.304042:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828167.304148:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828168.328020:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828168.328088:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828169.352042:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828170.376037:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828170.376120:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:3.0:1787828171.399934:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787828171.400087:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828172.296085:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828172.296100:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828172.296108:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828172.296127:0:27606:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1607 00000100:00080000:2.0:1787828172.296144:0:27606:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828172.296154:0:27606:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787828172.296234:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498d680 x1874672085545088/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828172.296270:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828172.296276:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828172.296280:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828172.296287:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1607 00000100:00080000:2.0:1787828172.296293:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828172.296299:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828172.296317:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828172.296338:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e940 x1874672085545216/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828172.296357:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828172.296361:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787828172.296365:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828172.296371:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1607 00000100:00080000:2.0:1787828172.296375:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828172.296379:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828172.296384:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828172.296400:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c3c0 x1874672085545344/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828172.296409:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828172.296413:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787828172.296415:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828172.296422:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1607 00000100:00080000:2.0:1787828172.296426:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828172.296430:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828172.296435:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:2.0:1787828172.296448:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498ed00 x1874672085545472/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828172.296469:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828172.296475:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787828172.296477:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828172.296484:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1607 00000100:00080000:2.0:1787828172.296489:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828172.296497:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828172.296502:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:2.0:1787828172.296522:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498de00 x1874672085545600/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828172.296543:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828172.296563:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828172.296571:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828172.296588:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1607 00000100:00080000:2.0:1787828172.296598:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828172.296614:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828172.296631:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828172.296667:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498fc00 x1874672085545728/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828172.296684:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828172.296693:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787828172.296697:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828172.296711:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1607 00000100:00080000:2.0:1787828172.296717:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828172.296738:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828172.296749:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828172.296786:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498d680 x1874672085545856/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828172.296814:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828172.296821:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787828172.296935:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828172.296965:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1607 00000100:00080000:2.0:1787828172.297000:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828172.297030:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828172.297053:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:2.0:1787828172.297097:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c780 x1874672085545984/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828172.297111:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828172.297116:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787828172.297121:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828172.297138:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1607 00000100:00080000:2.0:1787828172.297152:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828172.297166:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828172.297181:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:2.0:1787828172.297219:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cf00 x1874672085546112/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:3.0:1787828172.424056:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828173.447986:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787828173.448064:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828174.472007:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787828174.472078:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787828175.496052:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828176.520023:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787828176.520085:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787828177.544066:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787828178.568003:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828179.592036:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787828180.615994:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787828181.640042:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828182.536008:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828182.536027:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828182.536034:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828182.536054:0:27606:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1617 00000100:00080000:2.0:1787828182.536076:0:27606:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828182.536091:0:27606:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787828182.536164:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e1c0 x1874672085547392/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828182.536214:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828182.536220:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828182.536222:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828182.536228:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1617 00000100:00080000:2.0:1787828182.536233:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828182.536238:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828182.536255:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828182.536273:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f480 x1874672085547520/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828182.536290:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828182.536295:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787828182.536298:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828182.536309:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1617 00000100:00080000:2.0:1787828182.536319:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828182.536334:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828182.536346:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828182.536376:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f0c0 x1874672085547648/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828182.536398:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828182.536405:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787828182.536409:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828182.536415:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1617 00000100:00080000:2.0:1787828182.536422:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828182.536431:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828182.536443:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:2.0:1787828182.536465:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f840 x1874672085547776/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828182.536477:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828182.536483:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787828182.536486:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828182.536492:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1617 00000100:00080000:2.0:1787828182.536497:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828182.536503:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828182.536509:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:2.0:1787828182.536530:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cb40 x1874672085547904/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828182.536546:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828182.536566:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828182.536570:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828182.536580:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1617 00000100:00080000:2.0:1787828182.536589:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828182.536602:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828182.536636:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828182.536664:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c000 x1874672085548032/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828182.536680:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828182.536685:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787828182.536688:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828182.536695:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1617 00000100:00080000:2.0:1787828182.536701:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828182.536708:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828182.536727:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828182.536752:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e580 x1874672085548160/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828182.536769:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828182.536784:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787828182.536789:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828182.536889:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1617 00000100:00080000:2.0:1787828182.536902:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828182.536940:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828182.536963:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:2.0:1787828182.537001:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e1c0 x1874672085548288/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828182.537057:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828182.537064:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787828182.537069:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828182.537095:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1617 00000100:00080000:2.0:1787828182.537111:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828182.537131:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828182.537146:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:2.0:1787828182.537173:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cf00 x1874672085548416/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828182.664066:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828183.688013:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787828183.688094:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828184.712159:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828185.736027:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828185.736070:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828186.759952:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787828186.759960:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828187.656241:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085548544:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1787828187.656361:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085548672:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828187.656408:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085548800:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828187.656431:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085548928:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828187.656473:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085549056:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828187.656497:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085549184:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828187.656575:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085549312:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828187.656639:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085549440:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828187.656744:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085549568:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787828187.783895:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828188.807938:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828188.807960:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828189.832039:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828190.728411:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828187/real 1787828190] req@ffff9e35da718000 x1874672085549568/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828212 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828190.728495:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718000 x1874672085549568/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828212 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828190.728526:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828190.728536:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828190.728544:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085549568:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828190.728557:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828187/real 1787828190] req@ffff9e35da7192c0 x1874672085549440/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828212 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828190.728575:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7192c0 x1874672085549440/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828212 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828190.728587:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828190.728591:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828190.728596:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085549440:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828190.728604:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828187/real 1787828190] req@ffff9e35da718b40 x1874672085549312/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828212 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828190.728622:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718b40 x1874672085549312/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828212 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828190.728630:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828190.728633:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828190.728635:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085549312:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828190.728640:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828187/real 1787828190] req@ffff9e35da718f00 x1874672085549184/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828212 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828190.728649:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672085549184/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828212 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828190.728655:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828190.728658:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828190.728661:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085549184:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828190.728670:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828187/real 1787828190] req@ffff9e35da71a940 x1874672085548800/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828212 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828190.728685:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672085548800/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828212 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828190.728694:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828190.728699:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828190.728704:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085548800:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828190.728713:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828187/real 1787828190] req@ffff9e35da71a1c0 x1874672085548928/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828212 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828190.728729:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a1c0 x1874672085548928/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828212 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828190.728740:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828190.728744:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828190.728749:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085548928:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828190.728758:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828187/real 1787828190] req@ffff9e35da719a40 x1874672085549056/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828212 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828190.728774:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672085549056/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828212 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828190.728786:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828190.728791:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828190.728827:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085549056:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828190.728838:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828187/real 1787828190] req@ffff9e35da71a580 x1874672085548544/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828212 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828190.728882:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672085548544/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828212 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828190.728893:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828190.728913:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828190.728920:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085548544:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1787828190.728929:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828187/real 1787828190] req@ffff9e35da7183c0 x1874672085548672/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828212 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828190.728946:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672085548672/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828212 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828190.728956:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828190.728960:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828190.728964:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085548672:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828190.856029:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828191.879897:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828191.879960:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828192.776131:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085549696:192.168.203.141@tcp:250:kworker.0 00000800:00000100:2.0:1787828192.776243:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828192.776316:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828192/real 1787828192] req@ffff9e35da718780 x1874672085549696/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828217 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828192.776363:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718780 x1874672085549696/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828217 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828192.776412:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828192.776428:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828192.776450:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085549696:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1787828192.776511:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085549824:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828192.776560:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828192.776588:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828192/real 1787828192] req@ffff9e35da7183c0 x1874672085549824/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828217 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828192.776625:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672085549824/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828217 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828192.776650:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828192.776656:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828192.776668:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085549824:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828192.776695:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085549952:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828192.776744:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828192.776772:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828192/real 1787828192] req@ffff9e35da71a580 x1874672085549952/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828217 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828192.776893:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672085549952/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828217 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828192.776925:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828192.776939:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828192.776951:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085549952:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828192.776975:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085550080:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828192.777031:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828192.777058:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828192/real 1787828192] req@ffff9e35da719a40 x1874672085550080/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828217 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828192.777075:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672085550080/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828217 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828192.777084:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828192.777090:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828192.777095:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085550080:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828192.777105:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085550208:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828192.777128:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828192.777137:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828192/real 1787828192] req@ffff9e35da71a1c0 x1874672085550208/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828217 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828192.777146:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a1c0 x1874672085550208/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828217 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828192.777153:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828192.777155:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828192.777157:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085550208:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828192.777179:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085550336:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828192.777200:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828192.777211:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828192/real 1787828192] req@ffff9e35da71a940 x1874672085550336/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828217 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828192.777230:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672085550336/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828217 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828192.777239:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828192.777243:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828192.777246:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085550336:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828192.777253:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085550464:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828192.777275:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828192.777287:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828192/real 1787828192] req@ffff9e35da718f00 x1874672085550464/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828217 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828192.777299:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672085550464/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828217 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828192.777307:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828192.777309:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828192.777313:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085550464:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828192.777320:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085550592:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828192.777341:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828192.777359:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828192/real 1787828192] req@ffff9e35da718b40 x1874672085550592/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828217 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828192.777372:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718b40 x1874672085550592/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828217 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828192.777381:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828192.777385:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828192.777388:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085550592:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828192.777395:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085550720:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828192.777414:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828192.777424:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828192/real 1787828192] req@ffff9e35da7192c0 x1874672085550720/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828217 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828192.777435:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7192c0 x1874672085550720/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828217 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828192.777445:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828192.777448:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828192.777451:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085550720:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828192.903971:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828193.927915:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828194.952021:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828195.975936:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828195.975992:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828196.999946:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00080000:2.0:1787828197.895914:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828197.895925:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828197.895931:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828197.895942:0:27606:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1633 00000100:00080000:2.0:1787828197.895954:0:27606:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828197.895961:0:27606:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787828197.896003:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c780 x1874672085550848/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828197.896035:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828197.896040:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828197.896042:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828197.896048:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1633 00000100:00080000:2.0:1787828197.896054:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828197.896059:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828197.896072:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828197.896085:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498d680 x1874672085550976/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828197.896104:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828197.896111:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787828197.896113:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828197.896119:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1633 00000100:00080000:2.0:1787828197.896125:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828197.896131:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828197.896138:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828197.896151:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498fc00 x1874672085551104/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828197.896158:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828197.896161:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787828197.896162:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828197.896167:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1633 00000100:00080000:2.0:1787828197.896171:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828197.896173:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828197.896185:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:2.0:1787828197.896197:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498de00 x1874672085551232/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828197.896204:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828197.896208:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787828197.896209:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828197.896213:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1633 00000100:00080000:2.0:1787828197.896217:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828197.896221:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828197.896225:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:2.0:1787828197.896235:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498ed00 x1874672085551360/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828197.896257:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828197.896259:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828197.896263:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828197.896275:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1633 00000100:00080000:2.0:1787828197.896283:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828197.896288:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828197.896292:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828197.896307:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c3c0 x1874672085551488/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828197.896330:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828197.896338:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787828197.896340:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828197.896352:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1633 00000100:00080000:2.0:1787828197.896359:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828197.896371:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828197.896377:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828197.896388:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e940 x1874672085551616/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828197.896430:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828197.896433:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787828197.896437:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828197.896446:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1633 00000100:00080000:2.0:1787828197.896455:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828197.896462:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828197.896472:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:2.0:1787828197.896502:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c780 x1874672085551744/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828197.896549:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828197.896559:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787828197.896564:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828197.896576:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1633 00000100:00080000:2.0:1787828197.896586:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828197.896592:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828197.896597:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:2.0:1787828197.896613:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cf00 x1874672085551872/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828198.023953:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828199.047876:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828199.047927:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828200.071925:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828201.095943:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828202.119874:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828202.119922:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828203.015901:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828203.015910:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828203.015914:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828203.015921:0:27606:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1638 00000100:00080000:2.0:1787828203.015929:0:27606:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828203.015935:0:27606:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787828203.015965:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e1c0 x1874672085552000/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828203.015992:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828203.015997:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828203.015999:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828203.016003:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1638 00000100:00080000:2.0:1787828203.016007:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828203.016013:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828203.016024:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828203.016039:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e580 x1874672085552128/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828203.016056:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828203.016059:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787828203.016061:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828203.016072:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1638 00000100:00080000:2.0:1787828203.016075:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828203.016078:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828203.016083:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828203.016093:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c000 x1874672085552256/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828203.016101:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828203.016105:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787828203.016109:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828203.016115:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1638 00000100:00080000:2.0:1787828203.016120:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828203.016125:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828203.016131:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:2.0:1787828203.016147:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cb40 x1874672085552384/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828203.016162:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828203.016171:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787828203.016174:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828203.016179:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1638 00000100:00080000:2.0:1787828203.016185:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828203.016191:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828203.016198:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:2.0:1787828203.016216:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f840 x1874672085552512/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828203.016230:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828203.016233:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828203.016236:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828203.016241:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1638 00000100:00080000:2.0:1787828203.016244:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828203.016248:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828203.016255:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828203.016274:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f0c0 x1874672085552640/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828203.016288:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828203.016296:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787828203.016298:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828203.016307:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1638 00000100:00080000:2.0:1787828203.016312:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828203.016319:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828203.016324:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828203.016338:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f480 x1874672085552768/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828203.016352:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828203.016356:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787828203.016358:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828203.016367:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1638 00000100:00080000:2.0:1787828203.016373:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828203.016379:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828203.016386:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:2.0:1787828203.016411:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e1c0 x1874672085552896/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828203.016424:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828203.016428:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787828203.016433:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828203.016443:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1638 00000100:00080000:2.0:1787828203.016447:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828203.016453:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828203.016458:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:2.0:1787828203.016471:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cf00 x1874672085553024/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828203.143899:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828203.143939:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828204.167911:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828205.191870:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828205.191876:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828206.215936:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828206.215969:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828207.239901:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828207.239928:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828207.239937:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00080000:2.0:1787828208.135895:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828208.135905:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828208.135910:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828208.135922:0:27606:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1643 00000100:00080000:2.0:1787828208.135932:0:27606:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828208.135941:0:27606:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787828208.135975:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c780 x1874672085553152/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828208.136004:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828208.136009:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828208.136012:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828208.136017:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1643 00000100:00080000:2.0:1787828208.136022:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828208.136028:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828208.136038:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828208.136056:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e940 x1874672085553280/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828208.136080:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828208.136085:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787828208.136087:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828208.136092:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1643 00000100:00080000:2.0:1787828208.136097:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828208.136102:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828208.136108:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828208.136123:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c3c0 x1874672085553408/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828208.136137:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828208.136141:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787828208.136144:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828208.136148:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1643 00000100:00080000:2.0:1787828208.136152:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828208.136157:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828208.136163:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:2.0:1787828208.136178:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498ed00 x1874672085553536/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828208.136191:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828208.136199:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787828208.136203:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828208.136213:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1643 00000100:00080000:2.0:1787828208.136220:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828208.136229:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828208.136237:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:2.0:1787828208.136253:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498de00 x1874672085553664/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828208.136265:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828208.136268:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828208.136271:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828208.136276:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1643 00000100:00080000:2.0:1787828208.136282:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828208.136290:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828208.136295:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828208.136309:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498fc00 x1874672085553792/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828208.136325:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828208.136330:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787828208.136341:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828208.136351:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1643 00000100:00080000:2.0:1787828208.136358:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828208.136364:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828208.136372:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828208.136389:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498d680 x1874672085553920/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828208.136409:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828208.136418:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787828208.136421:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828208.136430:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1643 00000100:00080000:2.0:1787828208.136436:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828208.136442:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828208.136447:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:2.0:1787828208.136467:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c780 x1874672085554048/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828208.136481:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828208.136491:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787828208.136495:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828208.136504:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1643 00000100:00080000:2.0:1787828208.136511:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828208.136520:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828208.136526:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:2.0:1787828208.136544:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cf00 x1874672085554176/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828208.263914:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828208.263945:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787828209.287933:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787828210.311921:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828211.335902:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828212.359886:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828212.359902:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828213.255871:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828213.255878:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787828213.255882:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828213.255890:0:27606:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1648 00000100:00080000:3.0:1787828213.255899:0:27606:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828213.255904:0:27606:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787828213.255935:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a580 x1874672085554304/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828213.255955:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828213.255958:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828213.255959:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828213.255962:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1648 00000100:00080000:3.0:1787828213.255964:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828213.255966:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828213.255974:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828213.255984:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499680 x1874672085554432/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828213.255992:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828213.255996:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787828213.255999:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828213.256007:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1648 00000100:00080000:3.0:1787828213.256009:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828213.256011:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828213.256015:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828213.256023:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498780 x1874672085554560/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828213.256030:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828213.256032:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787828213.256034:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828213.256037:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1648 00000100:00080000:3.0:1787828213.256046:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828213.256048:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828213.256051:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:3.0:1787828213.256059:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498b40 x1874672085554688/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828213.256068:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828213.256072:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787828213.256074:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828213.256077:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1648 00000100:00080000:3.0:1787828213.256081:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828213.256083:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828213.256087:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:3.0:1787828213.256094:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a940 x1874672085554816/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828213.256100:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828213.256103:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828213.256106:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828213.256113:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1648 00000100:00080000:3.0:1787828213.256119:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828213.256124:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828213.256130:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828213.256147:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649bc00 x1874672085554944/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828213.256174:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828213.256181:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787828213.256184:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828213.256190:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1648 00000100:00080000:3.0:1787828213.256196:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828213.256199:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828213.256202:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828213.256209:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b840 x1874672085555072/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828213.256229:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828213.256232:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787828213.256233:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828213.256237:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1648 00000100:00080000:3.0:1787828213.256243:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828213.256248:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828213.256252:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:3.0:1787828213.256266:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498f00 x1874672085555200/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828213.256284:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828213.256289:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787828213.256292:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828213.256299:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1648 00000100:00080000:3.0:1787828213.256305:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828213.256311:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828213.256316:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:3.0:1787828213.256328:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a580 x1874672085555328/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828213.383903:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828213.383906:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828214.407890:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828215.431864:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828215.431930:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828216.455925:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828216.455942:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828217.479909:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787828217.479964:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828218.503890:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828219.527941:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828219.527962:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:3.0:1787828219.527989:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828220.551901:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828220.551957:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787828220.552060:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828221.575935:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828222.599922:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828222.599975:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828223.623885:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787828223.623937:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828224.647887:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828224.647940:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828225.671923:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828225.671964:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828226.695892:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828226.695898:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828227.719844:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828227.719926:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828228.615907:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828228.615915:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828228.615918:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828228.615926:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1663 00000100:00080000:2.0:1787828228.615935:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828228.615941:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787828228.615973:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e1c0 x1874672085557760/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828228.615995:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828228.615999:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828228.616000:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828228.616003:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1663 00000100:00080000:2.0:1787828228.616005:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828228.616008:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828228.616016:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828228.616024:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f480 x1874672085557888/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828228.616032:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828228.616034:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787828228.616036:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828228.616038:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1663 00000100:00080000:2.0:1787828228.616040:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828228.616042:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828228.616045:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828228.616055:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f0c0 x1874672085558016/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828228.616064:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828228.616066:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787828228.616067:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828228.616070:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1663 00000100:00080000:2.0:1787828228.616072:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828228.616075:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828228.616078:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:2.0:1787828228.616085:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f840 x1874672085558144/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828228.616091:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828228.616094:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787828228.616095:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828228.616097:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1663 00000100:00080000:2.0:1787828228.616100:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828228.616102:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828228.616105:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:2.0:1787828228.616114:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cb40 x1874672085558272/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828228.616123:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828228.616128:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828228.616130:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828228.616133:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1663 00000100:00080000:2.0:1787828228.616136:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828228.616139:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828228.616142:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828228.616148:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c000 x1874672085558400/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828228.616155:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828228.616157:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787828228.616159:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828228.616161:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1663 00000100:00080000:2.0:1787828228.616164:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828228.616166:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828228.616169:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828228.616180:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e580 x1874672085558528/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828228.616197:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828228.616200:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787828228.616201:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828228.616207:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1663 00000100:00080000:2.0:1787828228.616209:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828228.616221:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828228.616224:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:2.0:1787828228.616237:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cf00 x1874672085558656/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828228.616249:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828228.616253:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787828228.616254:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828228.616260:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1663 00000100:00080000:2.0:1787828228.616268:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828228.616273:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828228.616278:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:2.0:1787828228.616295:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c780 x1874672085558784/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828228.743899:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787828228.743963:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828229.767873:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828229.767920:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828230.791875:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828230.791926:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828231.815879:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828231.815894:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828232.839874:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787828232.839922:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828233.863859:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828233.863915:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828234.887938:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828234.888023:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828235.911951:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828235.912023:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828236.935882:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828236.935955:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828236.936050:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828237.959944:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828237.959988:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828237.960013:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828238.984056:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:3.0:1787828238.984140:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828240.007920:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:3.0:1787828240.007922:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828240.008002:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828241.031970:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828241.032027:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828241.032037:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828242.055922:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828243.080038:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828243.080061:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828244.103890:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828244.103947:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828245.127937:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828245.127974:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828246.152035:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828246.152035:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787828247.176025:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787828248.200016:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828248.200020:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828249.223958:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828250.247913:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828250.247948:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828251.271909:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828251.271954:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828251.271995:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:3.0:1787828252.295904:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828252.295913:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828253.319913:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828253.319934:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828253.319944:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828254.343910:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828254.343949:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828255.367969:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:3.0:1787828255.367999:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:3.0:1787828256.391978:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828256.391979:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828256.392082:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1787828257.288304:0:2397:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.141:988 00000400:00000100:2.0:1787828257.288451:0:2397:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.141@tcp at host 192.168.203.141 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1787828257.288478:0:2397:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:2.0:1787828257.288559:0:2397:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000800:00000100:2.0:1787828257.288584:0:2397:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:2.0:1787828257.288599:0:2397:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:2.0:1787828257.288614:0:2397:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:2.0:1787828257.288628:0:2397:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:2.0:1787828257.288642:0:2397:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:2.0:1787828257.288657:0:2397:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:2.0:1787828257.288672:0:2397:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:2.0:1787828257.416092:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828257.416157:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828258.439905:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828259.463864:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828259.463905:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828260.487945:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828260.488023:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828260.488051:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828261.511970:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828261.512054:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828262.535964:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828263.559981:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828263.560084:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828264.455969:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828264.455979:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828264.455983:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828264.455994:0:27606:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1699 00000100:00080000:2.0:1787828264.456004:0:27606:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828264.456011:0:27606:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787828264.456073:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498d680 x1874672085565824/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828264.456115:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828264.456120:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828264.456123:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828264.456128:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1699 00000100:00080000:2.0:1787828264.456133:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828264.456137:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828264.456150:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828264.456166:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498fc00 x1874672085565952/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828264.456180:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828264.456184:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787828264.456186:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828264.456202:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1699 00000100:00080000:2.0:1787828264.456208:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828264.456212:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828264.456220:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828264.456258:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498de00 x1874672085566080/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828264.456289:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828264.456292:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787828264.456293:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828264.456298:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1699 00000100:00080000:2.0:1787828264.456301:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828264.456305:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828264.456311:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:2.0:1787828264.456323:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498ed00 x1874672085566208/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828264.456332:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828264.456354:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787828264.456356:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828264.456390:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1699 00000100:00080000:2.0:1787828264.456401:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828264.456407:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828264.456415:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:2.0:1787828264.456429:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c3c0 x1874672085566336/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828264.456449:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828264.456455:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828264.456459:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828264.456468:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1699 00000100:00080000:2.0:1787828264.456476:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828264.456482:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828264.456489:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828264.456502:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e940 x1874672085566464/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828264.456519:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828264.456524:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787828264.456527:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828264.456536:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1699 00000100:00080000:2.0:1787828264.456547:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828264.456563:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828264.456582:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828264.456610:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c780 x1874672085566592/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828264.456649:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828264.456676:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787828264.456688:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828264.456702:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1699 00000100:00080000:2.0:1787828264.456711:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828264.456720:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828264.456733:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:2.0:1787828264.456752:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cf00 x1874672085566720/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828264.456762:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828264.456774:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787828264.456778:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828264.456784:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1699 00000100:00080000:2.0:1787828264.456789:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828264.456800:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828264.456814:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:2.0:1787828264.456892:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e580 x1874672085566848/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828264.583968:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828265.608031:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828266.631986:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787828266.632036:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828267.655895:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828267.655925:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828268.679957:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828268.679974:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828269.703890:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787828269.704084:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828270.727979:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828270.727995:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:3.0:1787828271.751886:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828271.751888:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828271.751943:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828272.775889:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828272.775949:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787828272.776017:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787828273.800048:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828273.800067:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00080000:2.0:1787828274.696005:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828274.696020:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828274.696027:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828274.696045:0:27606:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1710 00000100:00080000:2.0:1787828274.696068:0:27606:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828274.696080:0:27606:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787828274.696143:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c000 x1874672085568128/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828274.696185:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828274.696190:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828274.696193:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828274.696199:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1710 00000100:00080000:2.0:1787828274.696203:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828274.696208:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828274.696223:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828274.696241:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cb40 x1874672085568256/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828274.696257:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828274.696467:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787828274.696474:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828274.696485:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1710 00000100:00080000:2.0:1787828274.696498:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828274.696509:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828274.696519:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828274.696552:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f840 x1874672085568384/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828274.696605:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828274.696622:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787828274.696627:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828274.696638:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1710 00000100:00080000:2.0:1787828274.696647:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828274.696656:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828274.696668:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:2.0:1787828274.696705:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f0c0 x1874672085568512/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828274.696752:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828274.696763:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787828274.696767:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828274.696780:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1710 00000100:00080000:2.0:1787828274.696791:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828274.696867:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828274.696880:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:2.0:1787828274.696908:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f480 x1874672085568640/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828274.696926:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828274.696931:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828274.696936:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828274.696954:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1710 00000100:00080000:2.0:1787828274.696961:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828274.696966:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828274.696972:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828274.696992:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e1c0 x1874672085568768/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828274.697005:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828274.697011:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787828274.697017:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828274.697031:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1710 00000100:00080000:2.0:1787828274.697040:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828274.697044:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828274.697051:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828274.697069:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cb40 x1874672085568896/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828274.697082:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828274.697088:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787828274.697092:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828274.697104:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1710 00000100:00080000:2.0:1787828274.697113:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828274.697118:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828274.697125:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:2.0:1787828274.697145:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c000 x1874672085569024/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828274.697210:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828274.697224:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787828274.697230:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828274.697243:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1710 00000100:00080000:2.0:1787828274.697255:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828274.697313:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828274.697324:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:2.0:1787828274.697346:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e580 x1874672085569152/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828274.824001:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828274.824132:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828275.847948:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828275.848043:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787828276.871983:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828276.871985:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828276.872079:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828277.895961:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828277.896029:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828278.919952:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00080000:2.0:1787828279.815941:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828279.815954:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828279.815960:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828279.815983:0:27606:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1715 00000100:00080000:2.0:1787828279.816011:0:27606:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828279.816021:0:27606:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787828279.816083:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cf00 x1874672085569280/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828279.816130:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828279.816136:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828279.816141:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828279.816150:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1715 00000100:00080000:2.0:1787828279.816160:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828279.816167:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828279.816183:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828279.816215:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c780 x1874672085569408/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828279.816243:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828279.816249:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787828279.816253:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828279.816260:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1715 00000100:00080000:2.0:1787828279.816265:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828279.816271:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828279.816280:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828279.816305:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e940 x1874672085569536/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828279.816321:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828279.816330:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787828279.816340:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828279.816353:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1715 00000100:00080000:2.0:1787828279.816368:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828279.816378:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828279.816393:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:2.0:1787828279.816415:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c3c0 x1874672085569664/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828279.816447:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828279.816454:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787828279.816460:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828279.816489:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1715 00000100:00080000:2.0:1787828279.816504:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828279.816521:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828279.816534:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:2.0:1787828279.816570:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498ed00 x1874672085569792/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828279.816584:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828279.816588:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828279.816597:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828279.816610:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1715 00000100:00080000:2.0:1787828279.816620:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828279.816639:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828279.816660:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828279.816692:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498de00 x1874672085569920/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828279.816727:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828279.816735:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787828279.816743:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828279.816752:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1715 00000100:00080000:2.0:1787828279.816761:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828279.816771:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828279.816783:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828279.816843:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498fc00 x1874672085570048/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828279.816885:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828279.816891:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787828279.816894:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828279.816912:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1715 00000100:00080000:2.0:1787828279.816921:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828279.816936:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828279.816953:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:2.0:1787828279.816992:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498d680 x1874672085570176/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828279.817021:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828279.817025:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787828279.817039:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828279.817052:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1715 00000100:00080000:2.0:1787828279.817069:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828279.817083:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828279.817110:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:2.0:1787828279.817141:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e940 x1874672085570304/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828279.943936:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828279.943963:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828280.967936:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828280.967997:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828281.991964:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828283.015939:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787828283.016010:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828284.039918:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828284.039929:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828285.063907:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828286.087929:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828286.087960:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828287.111927:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828287.111977:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828288.135925:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828289.159917:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787828289.160030:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828290.183912:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828290.183940:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828291.207959:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828291.207967:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828292.231900:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787828292.231935:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828293.255875:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828293.255936:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828293.255945:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828294.279921:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828294.279970:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828295.303912:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828296.327926:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828296.327945:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828297.351898:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828297.351945:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828298.375927:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:3.0:1787828298.376001:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828299.399922:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828299.399976:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828300.423909:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828300.423951:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828301.447958:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828302.471945:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828304.519930:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828305.543940:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828306.567877:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787828306.567914:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828307.591901:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828307.591977:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828308.615922:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828308.615981:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828308.615990:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828309.639960:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:3.0:1787828310.663942:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828310.664091:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828311.687986:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828311.688045:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828312.711914:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828313.735936:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828313.736031:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828314.759861:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828314.759948:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828315.655921:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828315.655931:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787828315.655936:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828315.655948:0:27606:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1751 00000100:00080000:3.0:1787828315.655959:0:27606:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828315.655966:0:27606:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787828315.656014:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499a40 x1874672085577344/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828315.656041:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828315.656045:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828315.656047:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828315.656053:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1751 00000100:00080000:3.0:1787828315.656057:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828315.656062:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828315.656072:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828315.656090:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649ad00 x1874672085577472/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828315.656103:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828315.656107:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787828315.656110:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828315.656115:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1751 00000100:00080000:3.0:1787828315.656120:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828315.656125:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828315.656129:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828315.656140:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b0c0 x1874672085577600/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828315.656161:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828315.656164:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787828315.656165:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828315.656168:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1751 00000100:00080000:3.0:1787828315.656171:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828315.656174:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828315.656231:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:3.0:1787828315.656244:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b480 x1874672085577728/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828315.656266:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828315.656269:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787828315.656270:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828315.656274:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1751 00000100:00080000:3.0:1787828315.656276:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828315.656279:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828315.656283:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:3.0:1787828315.656292:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499e00 x1874672085577856/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828315.656300:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828315.656303:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828315.656305:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828315.656307:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1751 00000100:00080000:3.0:1787828315.656311:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828315.656313:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828315.656318:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828315.656441:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c64983c0 x1874672085577984/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828315.656513:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828315.656521:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787828315.656524:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828315.656528:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1751 00000100:00080000:3.0:1787828315.656532:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828315.656536:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828315.656541:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828315.656554:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a1c0 x1874672085578112/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828315.656574:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828315.656578:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787828315.656580:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828315.656588:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1751 00000100:00080000:3.0:1787828315.656596:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828315.656603:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828315.656610:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:3.0:1787828315.656626:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498000 x1874672085578240/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828315.656650:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828315.656657:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787828315.656658:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828315.656662:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1751 00000100:00080000:3.0:1787828315.656665:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828315.656669:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828315.656672:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:3.0:1787828315.656682:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c64992c0 x1874672085578368/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828315.783888:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828316.807927:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828316.808069:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828317.831903:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828317.831989:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828318.855921:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828319.879928:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828319.879944:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828320.776305:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5a40 x1874672085579392/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828320.776326:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828320.776330:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828320.776332:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828320.776336:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1756 00000100:00080000:1.0:1787828320.776341:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828320.776345:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828320.776350:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:1.0:1787828320.776361:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f61c0 x1874672085579520/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828320.903907:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787828320.904049:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787828320.904074:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828321.927925:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828321.927925:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787828322.951877:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828322.951887:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828322.951996:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1787828323.848098:0:2398:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.141:988 00000400:00000100:3.0:1787828323.848144:0:2398:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.141@tcp at host 192.168.203.141 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1787828323.848160:0:2398:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1787828323.848218:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000800:00000100:3.0:1787828323.848231:0:2398:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1787828323.848254:0:2398:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1787828323.848262:0:2398:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1787828323.848271:0:2398:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1787828323.848282:0:2398:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1787828323.848294:0:2398:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1787828323.848303:0:2398:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:2.0:1787828323.975883:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787828323.975897:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787828323.975971:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828324.999976:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787828325.000041:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787828325.000050:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828325.896178:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085579648:192.168.203.141@tcp:250:kworker.0 00000800:00000100:1.0:1787828325.896258:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828325.896328:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828325/real 1787828325] req@ffff9e35da718000 x1874672085579648/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828350 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828325.896385:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718000 x1874672085579648/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828350 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828325.896425:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828325.896436:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828325.896448:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085579648:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1787828325.896484:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085579776:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787828325.896532:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828325.896558:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828325/real 1787828325] req@ffff9e35da718780 x1874672085579776/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828350 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828325.896580:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718780 x1874672085579776/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828350 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828325.896602:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828325.896612:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828325.896623:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085579776:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828325.896639:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085579904:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787828325.896680:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828325.896703:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828325/real 1787828325] req@ffff9e35da719a40 x1874672085579904/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828350 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828325.896734:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672085579904/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828350 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828325.896752:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828325.896759:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828325.896768:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085579904:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828325.896786:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085580032:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828325.896846:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828325.896868:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828325/real 1787828325] req@ffff9e35da71a580 x1874672085580032/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828350 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828325.896901:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672085580032/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828350 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828325.896928:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828325.896937:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828325.896948:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085580032:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828325.896981:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085580160:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828325.897025:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828325.897047:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828325/real 1787828325] req@ffff9e35da7183c0 x1874672085580160/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828350 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828325.897064:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672085580160/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828350 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828325.897077:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828325.897081:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828325.897085:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085580160:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828325.897091:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085580288:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787828325.897113:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828325.897129:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828325/real 1787828325] req@ffff9e35da71a1c0 x1874672085580288/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828350 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828325.897145:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a1c0 x1874672085580288/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828350 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828325.897164:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828325.897168:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828325.897172:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085580288:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828325.897179:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085580416:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787828325.897203:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828325.897217:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828325/real 1787828325] req@ffff9e35da71a940 x1874672085580416/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828350 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828325.897232:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672085580416/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828350 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828325.897244:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828325.897248:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828325.897252:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085580416:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828325.897260:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085580544:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828325.897295:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828325.897310:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828325/real 1787828325] req@ffff9e35da718f00 x1874672085580544/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828350 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828325.897326:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672085580544/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828350 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828325.897337:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828325.897340:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828325.897343:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085580544:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828325.897348:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085580672:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828325.897365:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828325.897378:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828325/real 1787828325] req@ffff9e35da718b40 x1874672085580672/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828350 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828325.897394:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718b40 x1874672085580672/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828350 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828325.897404:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828325.897408:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828325.897412:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085580672:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828326.023886:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828326.023886:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828326.023951:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828327.047892:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787828327.048020:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787828327.048054:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828328.072009:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828328.072040:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828328.072061:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828329.095876:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828329.095948:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828329.095952:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828330.119902:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828330.119941:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828330.119965:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828331.016277:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085580800:192.168.203.141@tcp:250:kworker.0 00000800:00000100:1.0:1787828331.016408:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828331.016480:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828331/real 1787828331] req@ffff9e35da7192c0 x1874672085580800/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828356 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828331.016543:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7192c0 x1874672085580800/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828356 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828331.016583:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828331.016600:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828331.016615:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085580800:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1787828331.016673:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085580928:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787828331.016722:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828331.016755:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828331/real 1787828331] req@ffff9e35da718b40 x1874672085580928/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828356 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828331.016905:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718b40 x1874672085580928/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828356 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828331.016919:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828331.016925:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828331.016929:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085580928:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828331.016937:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085581056:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787828331.016966:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828331.016979:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828331/real 1787828331] req@ffff9e35da718f00 x1874672085581056/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828356 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828331.016992:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672085581056/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828356 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828331.017001:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828331.017005:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828331.017009:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085581056:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828331.017015:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085581184:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828331.017032:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828331.017042:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828331/real 1787828331] req@ffff9e35da71a940 x1874672085581184/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828356 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828331.017054:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672085581184/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828356 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828331.017062:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828331.017065:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828331.017068:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085581184:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828331.017074:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085581312:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828331.017090:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828331.017100:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828331/real 1787828331] req@ffff9e35da71a1c0 x1874672085581312/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828356 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828331.017112:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a1c0 x1874672085581312/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828356 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828331.017121:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828331.017124:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828331.017127:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085581312:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828331.017133:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085581440:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787828331.017149:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828331.017159:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828331/real 1787828331] req@ffff9e35da7183c0 x1874672085581440/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828356 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828331.017170:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672085581440/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828356 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828331.017180:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828331.017184:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828331.017187:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085581440:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828331.017212:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085581568:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787828331.017229:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828331.017240:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828331/real 1787828331] req@ffff9e35da71a580 x1874672085581568/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828356 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828331.017252:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672085581568/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828356 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828331.017261:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828331.017264:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828331.017267:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085581568:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828331.017273:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085581696:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828331.017289:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828331.017299:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828331/real 1787828331] req@ffff9e35da719a40 x1874672085581696/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828356 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828331.017322:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672085581696/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828356 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828331.017331:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828331.017334:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828331.017339:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085581696:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828331.017345:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085581824:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828331.017359:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828331.017368:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828331/real 1787828331] req@ffff9e35da718780 x1874672085581824/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828356 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828331.017387:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718780 x1874672085581824/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828356 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828331.017396:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828331.017399:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828331.017402:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085581824:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828331.144055:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828331.144119:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828331.144120:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828332.167949:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828332.168000:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828332.168018:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828332.168021:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828333.191968:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828333.192031:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828333.192136:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828334.215873:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828334.215891:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828334.215933:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828335.239889:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828335.239930:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828336.135995:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828336.136008:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828336.136014:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828336.136026:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1771 00000100:00080000:2.0:1787828336.136041:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828336.136050:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787828336.136108:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c780 x1874672085581952/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828336.136161:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828336.136169:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828336.136172:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828336.136181:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1771 00000100:00080000:2.0:1787828336.136190:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828336.136198:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828336.136212:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828336.136235:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085581952:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1787828336.136238:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cf00 x1874672085582080/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828336.136277:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828336.136283:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787828336.136286:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828336.136291:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1771 00000100:00080000:2.0:1787828336.136298:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828336.136302:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828336.136309:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828336.136329:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e580 x1874672085582208/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787828336.136341:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1787828336.136354:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828336.136359:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787828336.136362:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828336.136369:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1771 00000100:00080000:2.0:1787828336.136374:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828336.136379:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828336.136385:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:2.0:1787828336.136401:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c000 x1874672085582336/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828336.136413:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828336.136418:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787828336.136425:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787828336.136427:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828336/real 1787828336] req@ffff9e35c498c780 x1874672085581952/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828361 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828336.136435:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1771 00000100:00080000:2.0:1787828336.136444:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828336.136455:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828336.136467:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:1.0:1787828336.136479:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c780 x1874672085581952/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828361 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828336.136492:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cb40 x1874672085582464/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828336.136500:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828336.136531:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:2.0:1787828336.136533:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828336.136547:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:1.0:1787828336.136549:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085581952:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1787828336.136556:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828336.136573:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1771 00000100:00080000:2.0:1787828336.136581:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828336.136590:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828336.136600:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828336.136609:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085582080:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828336.136623:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e1c0 x1874672085582592/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828336.136640:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828336.136645:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787828336.136648:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828336.136653:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1771 00000100:00080000:2.0:1787828336.136660:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1787828336.136666:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1787828336.136679:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828336.136686:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828336.136698:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828336/real 1787828336] req@ffff9e35c498cf00 x1874672085582080/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828361 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828336.136707:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f480 x1874672085582720/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828336.136745:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787828336.136751:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498cf00 x1874672085582080/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828361 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828336.136753:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787828336.136760:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828336.136782:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1771 00000100:00080000:2.0:1787828336.136795:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828336.136902:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828336.136910:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00080000:1.0:1787828336.136913:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828336.136926:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828336.136932:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f0c0 x1874672085582848/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828336.136947:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828336.136951:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787828336.136955:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787828336.136959:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085582080:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1787828336.136964:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1771 00000100:00080000:2.0:1787828336.136975:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828336.136983:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787828336.136989:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085582208:192.168.203.141@tcp:38:kworker.0 00000008:00100000:2.0:1787828336.136993:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:2.0:1787828336.137014:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f840 x1874672085582976/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787828336.137038:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828336.137062:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828336/real 1787828336] req@ffff9e35c498e580 x1874672085582208/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828361 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828336.137078:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e580 x1874672085582208/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828361 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828336.137089:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828336.137092:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828336.137096:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085582208:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828336.137103:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085582336:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828336.137123:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828336.137133:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828336/real 1787828336] req@ffff9e35c498c000 x1874672085582336/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828361 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828336.137146:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c000 x1874672085582336/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828361 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828336.137155:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828336.137158:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828336.137162:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085582336:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828336.137168:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085582464:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828336.137185:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828336.137195:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828336/real 1787828336] req@ffff9e35c498cb40 x1874672085582464/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828361 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828336.137207:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498cb40 x1874672085582464/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828361 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828336.137216:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828336.137219:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828336.137223:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085582464:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828336.137247:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085582592:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787828336.137265:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828336.137276:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828336/real 1787828336] req@ffff9e35c498e1c0 x1874672085582592/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828361 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828336.137309:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e1c0 x1874672085582592/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828361 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828336.137326:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828336.137332:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828336.137339:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085582592:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828336.137350:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085582720:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787828336.137385:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828336.137400:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828336/real 1787828336] req@ffff9e35c498f480 x1874672085582720/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828361 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828336.137414:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f480 x1874672085582720/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828361 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828336.137423:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828336.137427:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828336.137431:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085582720:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828336.137437:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085582848:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828336.137457:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828336.137467:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828336/real 1787828336] req@ffff9e35c498f0c0 x1874672085582848/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828361 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828336.137479:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f0c0 x1874672085582848/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828361 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828336.137497:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828336.137500:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828336.137504:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085582848:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828336.137510:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085582976:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828336.137543:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828336.137560:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828336/real 1787828336] req@ffff9e35c498f840 x1874672085582976/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828361 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828336.137583:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f840 x1874672085582976/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828361 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828336.137600:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828336.137606:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828336.137629:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085582976:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828336.263996:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828336.264001:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787828336.264018:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828336.264120:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828337.287976:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828337.287984:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828337.288035:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787828337.288070:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828338.311982:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828338.311982:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828338.312011:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828338.312020:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828339.335927:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787828339.335936:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787828339.335948:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828339.336016:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828340.359924:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828340.360003:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828340.360021:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828340.360048:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787828341.256241:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085583104:192.168.203.141@tcp:250:kworker.0 00000800:00000100:1.0:1787828341.256367:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828341.256437:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828341/real 1787828341] req@ffff9e35da718000 x1874672085583104/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828366 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828341.256488:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718000 x1874672085583104/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828366 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828341.256517:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828341.256537:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828341.256557:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085583104:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1787828341.256621:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085583232:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787828341.256686:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828341.256719:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828341/real 1787828341] req@ffff9e35da718780 x1874672085583232/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828366 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828341.256774:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718780 x1874672085583232/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828366 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828341.256900:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828341.256912:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828341.256924:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085583232:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828341.256949:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085583360:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787828341.257007:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828341.257043:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828341/real 1787828341] req@ffff9e35da719a40 x1874672085583360/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828366 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828341.257088:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672085583360/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828366 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828341.257101:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828341.257107:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828341.257111:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085583360:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828341.257123:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085583488:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828341.257156:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828341.257174:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828341/real 1787828341] req@ffff9e35da71a580 x1874672085583488/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828366 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828341.257190:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672085583488/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828366 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828341.257202:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828341.257208:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828341.257213:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085583488:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828341.257223:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085583616:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828341.257248:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828341.257260:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828341/real 1787828341] req@ffff9e35da7183c0 x1874672085583616/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828366 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828341.257274:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672085583616/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828366 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828341.257284:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828341.257287:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828341.257291:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085583616:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828341.257316:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085583744:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787828341.257336:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828341.257349:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828341/real 1787828341] req@ffff9e35da71a1c0 x1874672085583744/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828366 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828341.257364:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a1c0 x1874672085583744/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828366 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828341.257374:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828341.257378:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828341.257381:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085583744:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828341.257388:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085583872:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787828341.257406:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828341.257417:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828341/real 1787828341] req@ffff9e35da71a940 x1874672085583872/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828366 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828341.257448:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672085583872/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828366 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828341.257457:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828341.257460:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828341.257463:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085583872:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828341.257469:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085584000:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828341.257486:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828341.257495:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828341/real 1787828341] req@ffff9e35da718f00 x1874672085584000/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828366 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828341.257507:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672085584000/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828366 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828341.257527:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828341.257533:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828341.257539:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085584000:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828341.257549:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085584128:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828341.257579:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828341.257598:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828341/real 1787828341] req@ffff9e35da718b40 x1874672085584128/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828366 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828341.257619:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718b40 x1874672085584128/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828366 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828341.257648:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828341.257652:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828341.257657:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085584128:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828341.383955:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787828341.383962:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828341.383999:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828341.384016:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828342.407894:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828342.407902:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787828342.407912:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828342.407977:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828343.431955:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828343.431972:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:3.0:1787828343.432014:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828343.432119:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828344.455905:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828344.455906:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787828344.455910:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828344.455980:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828345.479917:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828345.479926:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787828345.479926:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828345.480002:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828346.376189:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085584256:192.168.203.141@tcp:250:kworker.0 00000800:00000100:1.0:1787828346.376341:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828346.376434:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828346/real 1787828346] req@ffff9e35da7192c0 x1874672085584256/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828371 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828346.376485:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7192c0 x1874672085584256/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828371 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828346.376559:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828346.376571:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828346.376586:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085584256:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1787828346.376645:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085584384:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787828346.376734:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828346.376792:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828346/real 1787828346] req@ffff9e35da718b40 x1874672085584384/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828371 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828346.376919:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718b40 x1874672085584384/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828371 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828346.376955:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828346.376970:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828346.376981:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085584384:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828346.377017:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085584512:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787828346.377069:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828346.377086:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828346/real 1787828346] req@ffff9e35da718f00 x1874672085584512/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828371 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828346.377122:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672085584512/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828371 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828346.377140:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828346.377147:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828346.377153:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085584512:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828346.377166:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085584640:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828346.377208:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828346.377256:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828346/real 1787828346] req@ffff9e35da71a940 x1874672085584640/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828371 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828346.377272:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672085584640/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828371 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828346.377287:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828346.377304:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828346.377310:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085584640:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828346.377321:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085584768:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828346.377377:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828346.377400:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828346/real 1787828346] req@ffff9e35da71a1c0 x1874672085584768/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828371 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828346.377420:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a1c0 x1874672085584768/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828371 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828346.377432:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828346.377436:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828346.377440:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085584768:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828346.377466:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085584896:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787828346.377494:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828346.377507:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828346/real 1787828346] req@ffff9e35da7183c0 x1874672085584896/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828371 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828346.377552:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672085584896/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828371 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828346.377562:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828346.377566:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828346.377569:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085584896:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828346.377575:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085585024:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787828346.377594:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828346.377604:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828346/real 1787828346] req@ffff9e35da71a580 x1874672085585024/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828371 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828346.377617:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672085585024/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828371 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828346.377626:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828346.377629:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828346.377632:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085585024:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828346.377638:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085585152:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828346.377673:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828346.377694:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828346/real 1787828346] req@ffff9e35da719a40 x1874672085585152/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828371 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828346.377708:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672085585152/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828371 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828346.377728:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828346.377732:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828346.377736:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085585152:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828346.377742:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085585280:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828346.377759:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828346.377838:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828346/real 1787828346] req@ffff9e35da718780 x1874672085585280/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828371 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828346.377863:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718780 x1874672085585280/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828371 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828346.377880:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828346.377888:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828346.377904:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085585280:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828346.503899:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828346.503927:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:3.0:1787828346.503957:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828346.504097:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828347.527872:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828347.527899:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828347.527937:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828348.551922:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828348.551994:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828348.552047:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828349.575937:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828349.575953:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787828349.576025:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828350.599947:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828350.600008:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828351.495964:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828351.495977:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828351.495983:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828351.495998:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1786 00000100:00080000:1.0:1787828351.496011:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828351.496023:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787828351.496070:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7480 x1874672085585408/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828351.496117:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828351.496133:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828351.496136:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828351.496141:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1786 00000100:00080000:1.0:1787828351.496145:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828351.496149:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828351.496163:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828351.496188:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6d00 x1874672085585536/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828351.496206:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1787828351.496213:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085585408:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787828351.496214:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787828351.496220:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828351.496236:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1786 00000100:00080000:1.0:1787828351.496242:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828351.496246:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828351.496253:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828351.496272:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4b40 x1874672085585664/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828351.496290:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828351.496299:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828351.496303:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828351.496331:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1786 00000800:00000100:3.0:1787828351.496343:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787828351.496349:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828351.496364:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828351.496377:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:1.0:1787828351.496405:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f43c0 x1874672085585792/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828351.496423:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828351.496429:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828351.496433:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828351.496446:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1786 00000100:00080000:1.0:1787828351.496454:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828351.496461:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828351.496469:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:1.0:1787828351.496484:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4780 x1874672085585920/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828351.496496:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828351.496501:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828351.496503:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828351.496507:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1786 00000100:00080000:1.0:1787828351.496510:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828351.496514:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828351.496519:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828351.496528:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7840 x1874672085586048/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828351.496548:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828351.496553:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787828351.496555:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828351.496560:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1786 00000100:00080000:1.0:1787828351.496563:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828351.496567:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828351.496572:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828351.496585:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4000 x1874672085586176/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828351.496600:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1787828351.496600:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828351/real 1787828351] req@ffff9e35da6f7480 x1874672085585408/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828376 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828351.496607:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828351.496610:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828351.496614:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1786 00000100:00080000:1.0:1787828351.496618:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828351.496622:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828351.496627:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:1.0:1787828351.496636:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5e00 x1874672085586304/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828351.496647:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828351.496650:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828351.496653:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828351.496672:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1786 00000100:00080000:1.0:1787828351.496682:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828351.496687:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828351.496692:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:1.0:1787828351.496707:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4f00 x1874672085586432/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787828351.496912:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7480 x1874672085585408/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828376 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828351.496941:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828351.496948:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828351.496959:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085585408:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1787828351.497131:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085585536:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787828351.497174:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828351.497200:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828351/real 1787828351] req@ffff9e35da6f6d00 x1874672085585536/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828376 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828351.497226:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6d00 x1874672085585536/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828376 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828351.497244:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828351.497251:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828351.497258:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085585536:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787828351.497272:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085585664:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787828351.497316:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828351.497342:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828351/real 1787828351] req@ffff9e35da6f4b40 x1874672085585664/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828376 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828351.497368:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4b40 x1874672085585664/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828376 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828351.497400:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828351.497407:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828351.497413:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085585664:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787828351.497421:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085585792:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787828351.497452:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828351.497468:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828351/real 1787828351] req@ffff9e35da6f43c0 x1874672085585792/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828376 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828351.497483:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f43c0 x1874672085585792/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828376 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828351.497494:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828351.497498:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828351.497501:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085585792:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787828351.497508:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085585920:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787828351.497527:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828351.497538:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828351/real 1787828351] req@ffff9e35da6f4780 x1874672085585920/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828376 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828351.497551:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4780 x1874672085585920/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828376 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828351.497562:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828351.497565:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828351.497570:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085585920:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787828351.497577:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085586048:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787828351.497596:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828351.497607:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828351/real 1787828351] req@ffff9e35da6f7840 x1874672085586048/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828376 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828351.497620:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7840 x1874672085586048/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828376 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828351.497630:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828351.497634:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828351.497637:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085586048:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787828351.497644:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085586176:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787828351.497661:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828351.497672:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828351/real 1787828351] req@ffff9e35da6f4000 x1874672085586176/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828376 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828351.497686:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4000 x1874672085586176/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828376 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828351.497696:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828351.497699:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828351.497703:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085586176:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787828351.497709:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085586304:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787828351.497727:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828351.497781:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828351/real 1787828351] req@ffff9e35da6f5e00 x1874672085586304/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828376 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828351.497796:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5e00 x1874672085586304/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828376 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828351.497851:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828351.497859:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828351.497865:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085586304:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787828351.497876:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085586432:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787828351.497905:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828351.497919:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828351/real 1787828351] req@ffff9e35da6f4f00 x1874672085586432/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828376 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828351.497933:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4f00 x1874672085586432/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828376 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828351.497943:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828351.497948:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828351.497954:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085586432:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828351.623892:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828351.623967:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828351.623976:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787828352.647874:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828352.647911:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787828352.647921:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828353.671954:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828353.671989:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828353.672095:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828354.695941:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828354.695941:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828354.696012:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828355.719947:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828355.719956:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787828355.720129:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828356.616236:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085586560:192.168.203.141@tcp:250:kworker.0 00000800:00000100:3.0:1787828356.616370:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828356.616446:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828356/real 1787828356] req@ffff9e35da718000 x1874672085586560/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828381 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828356.616516:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718000 x1874672085586560/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828381 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828356.616565:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828356.616580:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828356.616591:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085586560:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1787828356.616664:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085586688:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787828356.616770:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828356.616902:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828356/real 1787828356] req@ffff9e35da718780 x1874672085586688/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828381 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828356.616947:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718780 x1874672085586688/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828381 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828356.616984:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828356.616990:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828356.617003:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085586688:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787828356.617023:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085586816:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787828356.617085:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828356.617109:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828356/real 1787828356] req@ffff9e35da719a40 x1874672085586816/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828381 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828356.617131:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672085586816/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828381 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828356.617199:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828356.617223:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828356.617236:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085586816:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787828356.617253:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085586944:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787828356.617315:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828356.617369:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828356/real 1787828356] req@ffff9e35da71a580 x1874672085586944/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828381 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828356.617393:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672085586944/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828381 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828356.617410:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828356.617414:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828356.617420:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085586944:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787828356.617428:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085587072:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787828356.617455:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828356.617468:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828356/real 1787828356] req@ffff9e35da7183c0 x1874672085587072/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828381 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828356.617482:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672085587072/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828381 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828356.617492:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828356.617497:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828356.617504:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085587072:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787828356.617556:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085587200:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787828356.617593:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828356.617633:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828356/real 1787828356] req@ffff9e35da71a1c0 x1874672085587200/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828381 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828356.617650:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a1c0 x1874672085587200/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828381 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828356.617661:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828356.617665:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828356.617669:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085587200:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787828356.617677:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085587328:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787828356.617745:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828356.617759:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828356/real 1787828356] req@ffff9e35da71a940 x1874672085587328/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828381 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828356.617773:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672085587328/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828381 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828356.617801:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828356.617805:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828356.617809:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085587328:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787828356.617847:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085587456:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787828356.617874:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828356.617907:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828356/real 1787828356] req@ffff9e35da718f00 x1874672085587456/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828381 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828356.617920:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672085587456/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828381 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828356.617930:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828356.617934:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828356.617937:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085587456:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787828356.617943:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085587584:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787828356.617980:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828356.617992:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828356/real 1787828356] req@ffff9e35da718b40 x1874672085587584/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828381 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828356.618006:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718b40 x1874672085587584/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828381 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828356.618016:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828356.618019:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828356.618023:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085587584:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828356.743974:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828356.743989:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828356.744125:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828357.767906:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828357.767916:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828357.767995:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828358.791850:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828358.791874:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787828358.791972:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828359.815877:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828359.815892:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828359.815903:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828359.815928:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828360.839881:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828360.839882:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828360.839906:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828361.736047:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085587712:192.168.203.141@tcp:250:kworker.0 00000800:00000100:3.0:1787828361.736112:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828361.736163:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828361/real 1787828361] req@ffff9e35da7192c0 x1874672085587712/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828386 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828361.736195:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7192c0 x1874672085587712/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828386 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828361.736208:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828361.736215:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828361.736221:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085587712:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1787828361.736231:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085587840:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787828361.736260:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828361.736275:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828361/real 1787828361] req@ffff9e35da718b40 x1874672085587840/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828386 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828361.736286:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718b40 x1874672085587840/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828386 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828361.736292:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828361.736295:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828361.736297:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085587840:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787828361.736302:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085587968:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787828361.736312:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828361.736318:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828361/real 1787828361] req@ffff9e35da718f00 x1874672085587968/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828386 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828361.736326:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672085587968/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828386 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828361.736335:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828361.736337:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828361.736340:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085587968:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787828361.736358:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085588096:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787828361.736368:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828361.736374:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828361/real 1787828361] req@ffff9e35da71a940 x1874672085588096/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828386 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828361.736381:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672085588096/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828386 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828361.736387:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828361.736389:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828361.736391:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085588096:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787828361.736395:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085588224:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787828361.736403:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828361.736409:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828361/real 1787828361] req@ffff9e35da71a1c0 x1874672085588224/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828386 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828361.736416:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a1c0 x1874672085588224/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828386 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828361.736421:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828361.736423:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828361.736425:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085588224:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787828361.736428:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085588352:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787828361.736438:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828361.736444:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828361/real 1787828361] req@ffff9e35da7183c0 x1874672085588352/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828386 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828361.736451:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672085588352/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828386 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828361.736456:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828361.736457:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828361.736459:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085588352:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787828361.736462:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085588480:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787828361.736472:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828361.736477:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828361/real 1787828361] req@ffff9e35da71a580 x1874672085588480/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828386 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828361.736485:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672085588480/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828386 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828361.736490:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828361.736491:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828361.736493:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085588480:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787828361.736496:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085588608:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787828361.736506:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828361.736512:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828361/real 1787828361] req@ffff9e35da719a40 x1874672085588608/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828386 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828361.736523:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672085588608/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828386 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828361.736532:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828361.736535:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828361.736538:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085588608:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787828361.736545:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085588736:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787828361.736565:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828361.736576:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828361/real 1787828361] req@ffff9e35da718780 x1874672085588736/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828386 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828361.736584:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718780 x1874672085588736/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828386 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828361.736590:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828361.736592:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828361.736594:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085588736:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828361.863867:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828361.863872:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828361.863907:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828362.887908:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828362.887918:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828362.887957:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828363.911882:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828363.911885:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828363.911922:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828364.935881:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828364.935897:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828364.935919:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828365.959892:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828365.959901:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828365.959960:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828366.855919:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828366.855929:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787828366.855934:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828366.855945:0:27606:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1802 00000100:00080000:3.0:1787828366.855955:0:27606:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828366.855963:0:27606:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787828366.856001:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c64992c0 x1874672085588864/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828366.856033:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828366.856036:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828366.856038:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828366.856042:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1802 00000100:00080000:3.0:1787828366.856044:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828366.856048:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828366.856068:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828366.856080:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498000 x1874672085588992/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828366.856090:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828366.856096:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:1.0:1787828366.856097:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c64992c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085588864:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1787828366.856098:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828366.856107:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1802 00000100:00080000:3.0:1787828366.856111:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828366.856113:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828366.856117:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828366.856130:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a1c0 x1874672085589120/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828366.856145:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828366.856148:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787828366.856149:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828366.856152:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1802 00000100:00080000:3.0:1787828366.856156:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828366.856159:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828366.856163:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000800:00000100:1.0:1787828366.856168:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828366.856175:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c64983c0 x1874672085589248/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828366.856185:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828366.856188:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787828366.856190:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828366.856193:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1802 00000100:00080000:3.0:1787828366.856196:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828366.856199:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828366.856203:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:1.0:1787828366.856213:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828366/real 1787828366] req@ffff9e35c64992c0 x1874672085588864/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828391 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828366.856216:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499e00 x1874672085589376/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828366.856252:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828366.856260:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828366.856282:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828366.856290:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1802 00000100:00080000:3.0:1787828366.856296:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828366.856306:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787828366.856307:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c64992c0 x1874672085588864/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828391 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1787828366.856314:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00080000:1.0:1787828366.856349:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1787828366.856353:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b480 x1874672085589504/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828366.856356:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828366.856364:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c64992c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085588864:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1787828366.856379:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828366.856383:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787828366.856386:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828366.856393:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1802 00000100:00100000:1.0:1787828366.856395:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085588992:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1787828366.856401:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828366.856420:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:1.0:1787828366.856423:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:3.0:1787828366.856432:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828366.856458:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828366/real 1787828366] req@ffff9e35c6498000 x1874672085588992/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828391 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828366.856473:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b0c0 x1874672085589632/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828366.856478:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498000 x1874672085588992/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828391 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828366.856539:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828366.856541:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828366.856562:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828366.856564:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:3.0:1787828366.856566:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787828366.856577:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085588992:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1787828366.856579:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1802 00000100:00080000:3.0:1787828366.856593:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1787828366.856601:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085589120:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1787828366.856607:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828366.856618:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:3.0:1787828366.856678:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649ad00 x1874672085589760/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828366.856693:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1787828366.856698:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1787828366.856699:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787828366.856705:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828366.856710:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1802 00000100:00080000:3.0:1787828366.856715:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828366.856718:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828366.856724:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:1.0:1787828366.856728:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828366/real 1787828366] req@ffff9e35c649a1c0 x1874672085589120/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828391 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828366.856737:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499a40 x1874672085589888/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828366.856763:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a1c0 x1874672085589120/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828391 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828366.856870:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828366.856874:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828366.856878:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085589120:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828366.856885:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c64983c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085589248:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828366.856911:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828366.856940:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828366/real 1787828366] req@ffff9e35c64983c0 x1874672085589248/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828391 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828366.856950:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c64983c0 x1874672085589248/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828391 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828366.856968:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828366.856971:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828366.856973:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c64983c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085589248:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828366.856978:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085589376:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828366.857010:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828366.857020:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828366/real 1787828366] req@ffff9e35c6499e00 x1874672085589376/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828391 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828366.857031:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499e00 x1874672085589376/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828391 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828366.857039:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828366.857042:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828366.857046:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085589376:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828366.857051:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085589504:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787828366.857066:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828366.857075:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828366/real 1787828366] req@ffff9e35c649b480 x1874672085589504/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828391 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828366.857086:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b480 x1874672085589504/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828391 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828366.857093:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828366.857096:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828366.857099:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085589504:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828366.857120:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085589632:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787828366.857136:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828366.857146:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828366/real 1787828366] req@ffff9e35c649b0c0 x1874672085589632/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828391 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828366.857173:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b0c0 x1874672085589632/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828391 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828366.857180:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828366.857183:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828366.857185:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085589632:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828366.857189:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085589760:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828366.857202:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828366.857210:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828366/real 1787828366] req@ffff9e35c649ad00 x1874672085589760/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828391 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828366.857219:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649ad00 x1874672085589760/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828391 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828366.857225:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828366.857227:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828366.857229:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085589760:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828366.857233:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085589888:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828366.857244:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828366.857251:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828366/real 1787828366] req@ffff9e35c6499a40 x1874672085589888/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828391 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828366.857260:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499a40 x1874672085589888/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828391 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828366.857285:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828366.857287:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828366.857290:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085589888:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828366.983880:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828366.983899:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828366.983928:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787828366.983943:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828368.007960:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787828368.007968:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828368.008010:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828369.031957:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787828369.032005:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828369.032019:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828370.055959:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:1.0:1787828370.056053:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828371.079919:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828371.079977:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828371.080001:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787828371.975957:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828371.975991:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787828371.976000:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828371.976020:0:27606:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1807 00000100:00080000:3.0:1787828371.976042:0:27606:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828371.976066:0:27606:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787828371.976153:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a580 x1874672085590016/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828371.976217:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828371.976224:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828371.976228:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828371.976242:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1807 00000100:00080000:3.0:1787828371.976262:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828371.976274:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828371.976295:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828371.976310:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085590016:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1787828371.976329:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498f00 x1874672085590144/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828371.976357:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828371.976362:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787828371.976365:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828371.976371:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1807 00000100:00080000:3.0:1787828371.976377:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828371.976383:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828371.976392:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828371.976430:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b840 x1874672085590272/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787828371.976450:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1787828371.976456:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828371.976478:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787828371.976484:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828371.976495:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1807 00000100:00080000:3.0:1787828371.976504:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828371.976511:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828371.976522:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:1.0:1787828371.976546:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828371/real 1787828371] req@ffff9e35c649a580 x1874672085590016/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828396 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828371.976557:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649bc00 x1874672085590400/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828371.976585:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787828371.976617:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a580 x1874672085590016/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828396 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828371.976621:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787828371.976628:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828371.976647:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1807 00000100:00080000:1.0:1787828371.976653:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828371.976655:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828371.976669:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:3.0:1787828371.976671:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787828371.976683:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085590016:192.168.203.141@tcp:250:kworker.0 00000008:00100000:3.0:1787828371.976684:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:3.0:1787828371.976715:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a940 x1874672085590528/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828371.976732:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787828371.976737:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085590144:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1787828371.976740:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828371.976750:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828371.976760:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1807 00000100:00080000:3.0:1787828371.976767:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828371.976772:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828371.976782:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1787828371.976788:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828371.976845:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498b40 x1874672085590656/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828371.976863:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828371.976868:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787828371.976872:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787828371.976880:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828371/real 1787828371] req@ffff9e35c6498f00 x1874672085590144/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828396 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828371.976882:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1807 00000100:00080000:3.0:1787828371.976896:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828371.976909:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787828371.976917:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498f00 x1874672085590144/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828396 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1787828371.976921:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00080000:1.0:1787828371.976947:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1787828371.976955:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498780 x1874672085590784/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828371.976961:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828371.976979:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085590144:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1787828371.977000:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787828371.977004:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085590272:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1787828371.977016:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787828371.977024:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828371.977036:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1807 00000100:00080000:3.0:1787828371.977046:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828371.977056:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828371.977067:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000800:00000100:1.0:1787828371.977089:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828371.977099:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499680 x1874672085590912/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828371.977116:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828371/real 1787828371] req@ffff9e35c649b840 x1874672085590272/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828396 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828371.977136:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828371.977155:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787828371.977161:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787828371.977177:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b840 x1874672085590272/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828396 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828371.977178:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1807 00000100:00080000:3.0:1787828371.977192:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828371.977204:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828371.977210:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:1.0:1787828371.977217:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828371.977227:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085590272:192.168.203.141@tcp:38:kworker.0 00000008:00100000:3.0:1787828371.977231:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:1.0:1787828371.977241:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085590400:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787828371.977273:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a580 x1874672085591040/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787828371.977288:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828371.977309:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828371/real 1787828371] req@ffff9e35c649bc00 x1874672085590400/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828396 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828371.977333:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649bc00 x1874672085590400/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828396 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828371.977345:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828371.977350:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828371.977355:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085590400:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828371.977387:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085590528:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828371.977408:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828371.977421:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828371/real 1787828371] req@ffff9e35c649a940 x1874672085590528/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828396 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828371.977435:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a940 x1874672085590528/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828396 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828371.977454:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828371.977458:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828371.977462:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085590528:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828371.977469:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085590656:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787828371.977487:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828371.977499:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828371/real 1787828371] req@ffff9e35c6498b40 x1874672085590656/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828396 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828371.977513:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498b40 x1874672085590656/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828396 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828371.977523:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828371.977527:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828371.977530:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085590656:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828371.977537:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085590784:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787828371.977555:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828371.977566:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828371/real 1787828371] req@ffff9e35c6498780 x1874672085590784/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828396 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828371.977580:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498780 x1874672085590784/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828396 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828371.977617:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828371.977621:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828371.977626:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085590784:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828371.977634:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085590912:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828371.977661:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828371.977675:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828371/real 1787828371] req@ffff9e35c6499680 x1874672085590912/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828396 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828371.977689:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499680 x1874672085590912/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828396 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828371.977700:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828371.977703:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828371.977707:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085590912:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828371.977714:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085591040:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828371.977732:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828371.977743:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828371/real 1787828371] req@ffff9e35c649a580 x1874672085591040/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828396 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828371.977757:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a580 x1874672085591040/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828396 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828371.977767:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828371.977770:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828371.977774:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085591040:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828372.103990:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828372.104039:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828372.104111:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828373.127881:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828373.127881:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:1.0:1787828373.127980:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828374.151866:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828374.151899:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828374.151935:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828375.175939:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828375.175942:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828375.176047:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828376.199915:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828376.199925:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828376.199970:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787828377.096177:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085591168:192.168.203.141@tcp:250:kworker.0 00000800:00000100:1.0:1787828377.096298:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828377.096362:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828377/real 1787828377] req@ffff9e35da718000 x1874672085591168/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828402 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828377.096430:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718000 x1874672085591168/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828402 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828377.096462:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828377.096478:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828377.096490:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085591168:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1787828377.096537:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085591296:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787828377.096602:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828377.096634:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828377/real 1787828377] req@ffff9e35da718780 x1874672085591296/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828402 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828377.096664:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718780 x1874672085591296/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828402 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828377.096709:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828377.096717:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828377.096728:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085591296:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828377.096748:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085591424:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787828377.096784:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828377.096836:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828377/real 1787828377] req@ffff9e35da719a40 x1874672085591424/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828402 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828377.096853:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672085591424/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828402 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828377.096870:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828377.096875:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828377.096882:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085591424:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828377.096896:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085591552:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828377.096937:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828377.096954:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828377/real 1787828377] req@ffff9e35da71a580 x1874672085591552/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828402 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828377.096967:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672085591552/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828402 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828377.096976:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828377.096980:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828377.096983:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085591552:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828377.096989:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085591680:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828377.097006:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828377.097015:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828377/real 1787828377] req@ffff9e35da7183c0 x1874672085591680/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828402 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828377.097035:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672085591680/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828402 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828377.097043:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828377.097046:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828377.097050:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085591680:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828377.097056:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085591808:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787828377.097079:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828377.097096:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828377/real 1787828377] req@ffff9e35da71a1c0 x1874672085591808/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828402 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828377.097117:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a1c0 x1874672085591808/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828402 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828377.097132:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828377.097138:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828377.097144:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085591808:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828377.097176:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085591936:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787828377.097209:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828377.097230:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828377/real 1787828377] req@ffff9e35da71a940 x1874672085591936/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828402 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828377.097251:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672085591936/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828402 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828377.097276:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828377.097280:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828377.097283:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085591936:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828377.097290:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085592064:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828377.097314:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828377.097326:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828377/real 1787828377] req@ffff9e35da718f00 x1874672085592064/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828402 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828377.097338:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672085592064/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828402 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828377.097347:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828377.097351:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828377.097354:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085592064:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828377.097359:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085592192:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828377.097375:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828377.097384:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828377/real 1787828377] req@ffff9e35da718b40 x1874672085592192/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828402 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828377.097395:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718b40 x1874672085592192/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828402 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828377.097404:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828377.097407:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828377.097410:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085592192:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828377.223959:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828377.223959:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787828377.224007:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828377.224090:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828378.247924:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828378.247953:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828378.247971:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828378.247972:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828379.271885:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787828379.271890:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828379.271917:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787828380.295898:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828380.295898:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787828380.295898:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828380.295967:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828381.319912:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:3.0:1787828381.319912:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828381.319916:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828381.319993:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828382.215926:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828382.215939:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828382.215943:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828382.215955:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1817 00000100:00080000:1.0:1787828382.215969:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828382.215977:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787828382.216019:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5680 x1874672085592320/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828382.216057:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828382.216068:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828382.216071:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828382.216077:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1817 00000100:00080000:1.0:1787828382.216082:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828382.216088:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828382.216101:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828382.216121:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f70c0 x1874672085592448/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787828382.216126:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085592320:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787828382.216157:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828382.216163:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787828382.216166:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828382.216173:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1817 00000100:00080000:1.0:1787828382.216179:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828382.216186:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828382.216194:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828382.216213:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6940 x1874672085592576/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787828382.216226:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787828382.216234:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828382.216241:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828382.216244:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828382.216253:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1817 00000100:00080000:1.0:1787828382.216259:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828382.216265:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828382.216273:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:3.0:1787828382.216276:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828382/real 1787828382] req@ffff9e35da6f5680 x1874672085592320/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828407 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828382.216293:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6580 x1874672085592704/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787828382.216299:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5680 x1874672085592320/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828407 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828382.216323:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828382.216333:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828382.216340:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828382.216343:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828382.216349:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1817 00000100:00080000:3.0:1787828382.216354:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:1.0:1787828382.216371:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1787828382.216372:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085592320:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787828382.216383:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828382.216391:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:3.0:1787828382.216406:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085592448:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828382.216411:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5a40 x1874672085592832/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828382.216430:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828382.216447:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000800:00000100:3.0:1787828382.216448:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787828382.216453:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828382.216463:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1817 00000100:00100000:3.0:1787828382.216467:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828382/real 1787828382] req@ffff9e35da6f70c0 x1874672085592448/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828407 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828382.216471:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828382.216484:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787828382.216490:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f70c0 x1874672085592448/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828407 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1787828382.216494:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00080000:3.0:1787828382.216508:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828382.216513:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828382.216539:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085592448:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828382.216540:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f61c0 x1874672085592960/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787828382.216561:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085592576:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787828382.216575:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828382.216580:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787828382.216584:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828382.216592:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1817 00000100:00080000:1.0:1787828382.216598:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1787828382.216603:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787828382.216609:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828382.216628:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828382.216634:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828382/real 1787828382] req@ffff9e35da6f6940 x1874672085592576/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828407 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828382.216652:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5680 x1874672085593088/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787828382.216657:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6940 x1874672085592576/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828407 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828382.216692:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828382.216699:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787828382.216699:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828382.216704:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828382.216711:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:1.0:1787828382.216717:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1817 00000100:00100000:3.0:1787828382.216725:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085592576:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787828382.216726:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828382.216738:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787828382.216739:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085592704:192.168.203.141@tcp:8:kworker.0 00000008:00100000:1.0:1787828382.216752:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000800:00000100:3.0:1787828382.216775:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828382.216788:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4f00 x1874672085593216/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787828382.216792:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828382/real 1787828382] req@ffff9e35da6f6580 x1874672085592704/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828407 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828382.216873:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1787828382.216874:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6580 x1874672085592704/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828407 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828382.216887:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828382.216893:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828382.216896:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828382.216904:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1817 00000100:00080000:3.0:1787828382.216905:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:1.0:1787828382.216916:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828382.216926:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787828382.216927:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085592704:192.168.203.141@tcp:8:kworker.0 00000008:00100000:1.0:1787828382.216941:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:3.0:1787828382.216957:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085592832:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828382.216967:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5e00 x1874672085593344/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787828382.216995:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828382.217010:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828382/real 1787828382] req@ffff9e35da6f5a40 x1874672085592832/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828407 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828382.217025:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5a40 x1874672085592832/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828407 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828382.217037:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828382.217042:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828382.217046:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085592832:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787828382.217055:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085592960:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787828382.217083:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828382.217108:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828382/real 1787828382] req@ffff9e35da6f61c0 x1874672085592960/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828407 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828382.217119:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f61c0 x1874672085592960/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828407 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828382.217126:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828382.217130:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828382.217132:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085592960:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787828382.217137:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085593088:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787828382.217158:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828382.217174:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828382/real 1787828382] req@ffff9e35da6f5680 x1874672085593088/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828407 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828382.217190:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5680 x1874672085593088/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828407 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828382.217202:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828382.217206:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828382.217211:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085593088:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787828382.217220:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085593216:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787828382.217241:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828382.217250:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828382/real 1787828382] req@ffff9e35da6f4f00 x1874672085593216/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828407 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828382.217261:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4f00 x1874672085593216/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828407 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828382.217268:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828382.217271:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828382.217274:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085593216:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787828382.217289:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085593344:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787828382.217303:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828382.217310:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828382/real 1787828382] req@ffff9e35da6f5e00 x1874672085593344/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828407 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828382.217320:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5e00 x1874672085593344/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828407 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828382.217327:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828382.217330:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828382.217333:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085593344:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828382.343876:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828382.343935:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828382.343938:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828382.343975:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828383.367911:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:3.0:1787828383.367937:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828383.368070:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828384.391987:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828384.392001:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828384.392044:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828385.415960:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828385.415996:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828386.439871:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828386.439920:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828386.439997:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828387.335915:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828387.335929:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828387.335936:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828387.335949:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1822 00000100:00080000:1.0:1787828387.335962:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828387.335973:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787828387.336018:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4000 x1874672085593472/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828387.336060:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828387.336066:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828387.336069:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828387.336077:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1822 00000100:00080000:1.0:1787828387.336096:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828387.336110:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828387.336126:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828387.336157:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7840 x1874672085593600/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787828387.336168:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085593472:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787828387.336200:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828387.336205:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787828387.336209:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828387.336218:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1822 00000100:00080000:1.0:1787828387.336228:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828387.336237:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828387.336247:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828387.336274:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4780 x1874672085593728/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828387.336292:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828387.336298:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828387.336300:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828387.336308:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1822 00000100:00080000:1.0:1787828387.336320:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828387.336326:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828387.336336:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:1.0:1787828387.336355:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f43c0 x1874672085593856/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828387.336369:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828387.336375:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828387.336378:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828387.336389:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1822 00000100:00080000:1.0:1787828387.336397:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828387.336405:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828387.336414:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:3.0:1787828387.336438:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085593600:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828387.336450:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4b40 x1874672085593984/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828387.336476:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828387.336501:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828387.336503:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828387.336510:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1822 00000100:00080000:1.0:1787828387.336517:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828387.336524:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828387.336533:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828387.336562:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6d00 x1874672085594112/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828387.336577:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828387.336581:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787828387.336585:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828387.336596:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1822 00000100:00080000:1.0:1787828387.336603:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828387.336610:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828387.336618:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828387.336640:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7480 x1874672085594240/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828387.336655:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828387.336662:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828387.336666:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828387.336674:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1822 00000100:00080000:1.0:1787828387.336684:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828387.336691:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828387.336699:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:1.0:1787828387.336720:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5e00 x1874672085594368/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828387.336743:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828387.336749:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828387.336752:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828387.336760:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1822 00000100:00080000:1.0:1787828387.336766:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828387.336772:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828387.336780:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:1.0:1787828387.336859:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4f00 x1874672085594496/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787828387.337015:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085593728:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787828387.337087:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085593856:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787828387.337147:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085593984:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787828387.337195:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085594112:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787828387.337240:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085594240:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787828387.337287:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085594368:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787828387.337331:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085594496:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828387.463935:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787828387.463971:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828387.464019:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828388.487933:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787828388.487966:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:1.0:1787828388.488027:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828389.511872:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787828389.511902:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828389.512082:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1787828390.408048:0:2399:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.141:988 00000400:00000100:2.0:1787828390.408092:0:2399:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.141@tcp at host 192.168.203.141 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1787828390.408106:0:2399:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:2.0:1787828390.408149:0:2399:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000800:00000100:2.0:1787828390.408159:0:2399:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:2.0:1787828390.408167:0:2399:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:2.0:1787828390.408181:0:2399:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:2.0:1787828390.408191:0:2399:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:2.0:1787828390.408198:0:2399:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:2.0:1787828390.408209:0:2399:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:3.0:1787828390.408214:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828387/real 1787828390] req@ffff9e35da6f4b40 x1874672085593984/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828412 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:2.0:1787828390.408218:0:2399:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:3.0:1787828390.408252:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4b40 x1874672085593984/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828412 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828390.408268:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828390.408273:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828390.408279:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085593984:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787828390.408289:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828387/real 1787828390] req@ffff9e35da6f6d00 x1874672085594112/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828412 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828390.408306:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6d00 x1874672085594112/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828412 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828390.408318:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828390.408322:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828390.408327:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085594112:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787828390.408335:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828387/real 1787828390] req@ffff9e35da6f7480 x1874672085594240/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828412 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828390.408349:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7480 x1874672085594240/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828412 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828390.408358:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828390.408362:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828390.408366:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085594240:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787828390.408373:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828387/real 1787828390] req@ffff9e35da6f5e00 x1874672085594368/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828412 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828390.408396:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5e00 x1874672085594368/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828412 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828390.408406:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828390.408410:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828390.408414:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085594368:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787828390.408421:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828387/real 1787828390] req@ffff9e35da6f4f00 x1874672085594496/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828412 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828390.408435:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4f00 x1874672085594496/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828412 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828390.408445:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828390.408449:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828390.408452:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085594496:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787828390.408485:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828387/real 1787828390] req@ffff9e35da6f7840 x1874672085593600/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828412 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828390.408503:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7840 x1874672085593600/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828412 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828390.408514:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828390.408519:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828390.408523:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085593600:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787828390.408532:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828387/real 1787828390] req@ffff9e35da6f4780 x1874672085593728/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828412 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828390.408548:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4780 x1874672085593728/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828412 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828390.408559:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828390.408562:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828390.408567:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085593728:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787828390.408576:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828387/real 1787828390] req@ffff9e35da6f43c0 x1874672085593856/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828412 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828390.408587:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f43c0 x1874672085593856/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828412 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828390.408598:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828390.408602:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828390.408606:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085593856:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787828390.408615:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828387/real 1787828390] req@ffff9e35da6f4000 x1874672085593472/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828412 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828390.408631:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4000 x1874672085593472/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828412 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828390.408641:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828390.408644:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828390.408649:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085593472:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1787828390.535902:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828390.535903:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828390.535912:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828390.535978:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828391.559871:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787828391.560020:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828391.560023:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:3.0:1787828392.456288:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085594624:192.168.203.141@tcp:250:kworker.0 00000800:00000100:3.0:1787828392.456440:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828392.456575:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828392/real 1787828392] req@ffff9e35da7192c0 x1874672085594624/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828417 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828392.456631:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7192c0 x1874672085594624/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828417 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828392.456655:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828392.456671:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828392.456688:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085594624:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1787828392.456713:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085594752:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787828392.456778:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828392.456905:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828392/real 1787828392] req@ffff9e35da718b40 x1874672085594752/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828417 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828392.456948:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718b40 x1874672085594752/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828417 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828392.456980:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828392.456991:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828392.457000:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085594752:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787828392.457072:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085594880:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787828392.457133:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828392.457688:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828392/real 1787828392] req@ffff9e35da718f00 x1874672085594880/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828417 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828392.457724:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672085594880/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828417 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828392.457741:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828392.457747:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828392.457753:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085594880:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787828392.457765:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085595008:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787828392.457958:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828392.457987:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828392/real 1787828392] req@ffff9e35da71a940 x1874672085595008/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828417 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828392.458009:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672085595008/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828417 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828392.458038:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828392.458046:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828392.458054:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085595008:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787828392.458069:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085595136:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787828392.458113:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828392.458133:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828392/real 1787828392] req@ffff9e35da71a1c0 x1874672085595136/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828417 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828392.458150:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a1c0 x1874672085595136/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828417 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828392.458169:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828392.458174:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828392.458177:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085595136:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787828392.458184:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085595264:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787828392.458207:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828392.458218:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828392/real 1787828392] req@ffff9e35da7183c0 x1874672085595264/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828417 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828392.458231:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672085595264/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828417 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828392.458242:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828392.458248:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828392.458253:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085595264:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787828392.458263:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085595392:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787828392.458296:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828392.458328:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828392/real 1787828392] req@ffff9e35da71a580 x1874672085595392/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828417 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828392.458350:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672085595392/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828417 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828392.458365:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828392.458378:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828392.458383:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085595392:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787828392.458391:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085595520:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787828392.458420:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828392.458439:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828392/real 1787828392] req@ffff9e35da719a40 x1874672085595520/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828417 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828392.458482:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672085595520/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828417 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828392.458499:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828392.458504:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828392.458510:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085595520:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787828392.458547:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085595648:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787828392.458579:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828392.458598:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828392/real 1787828392] req@ffff9e35da718780 x1874672085595648/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828417 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828392.458618:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718780 x1874672085595648/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828417 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828392.458633:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828392.458639:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828392.458644:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085595648:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787828392.583964:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828392.583969:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828392.584068:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828393.607932:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:3.0:1787828393.607967:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828393.608062:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828394.631946:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787828394.632028:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787828394.632092:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787828395.655976:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828395.656189:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787828395.656197:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828396.679960:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787828396.679972:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828396.680030:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828396.680033:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:3.0:1787828397.576039:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085595776:192.168.203.141@tcp:250:kworker.0 00000800:00000100:3.0:1787828397.576100:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828397.576149:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828397/real 1787828397] req@ffff9e35da718000 x1874672085595776/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828422 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828397.576176:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718000 x1874672085595776/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828422 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828397.576194:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828397.576205:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828397.576212:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085595776:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1787828397.576240:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085595904:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787828397.576267:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828397.576282:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828397/real 1787828397] req@ffff9e35da718780 x1874672085595904/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828422 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828397.576308:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718780 x1874672085595904/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828422 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828397.576325:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828397.576331:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828397.576337:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085595904:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787828397.576351:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085596032:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787828397.576378:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828397.576391:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828397/real 1787828397] req@ffff9e35da719a40 x1874672085596032/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828422 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828397.576427:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672085596032/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828422 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828397.576451:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828397.576460:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828397.576468:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085596032:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787828397.576481:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085596160:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787828397.576507:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828397.576519:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828397/real 1787828397] req@ffff9e35da71a580 x1874672085596160/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828422 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828397.576528:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672085596160/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828422 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828397.576534:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828397.576537:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828397.576539:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085596160:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787828397.576551:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085596288:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787828397.576570:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828397.576578:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828397/real 1787828397] req@ffff9e35da7183c0 x1874672085596288/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828422 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828397.576591:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672085596288/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828422 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828397.576598:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828397.576600:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828397.576602:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085596288:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787828397.576606:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085596416:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787828397.576617:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828397.576629:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828397/real 1787828397] req@ffff9e35da71a1c0 x1874672085596416/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828422 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828397.576638:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a1c0 x1874672085596416/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828422 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828397.576644:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828397.576646:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828397.576648:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085596416:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787828397.576652:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085596544:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787828397.576663:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828397.576670:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828397/real 1787828397] req@ffff9e35da71a940 x1874672085596544/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828422 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828397.576677:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672085596544/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828422 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828397.576688:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828397.576690:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828397.576692:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085596544:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787828397.576695:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085596672:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787828397.576705:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828397.576711:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828397/real 1787828397] req@ffff9e35da718f00 x1874672085596672/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828422 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828397.576718:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672085596672/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828422 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828397.576724:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828397.576726:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828397.576729:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085596672:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787828397.576733:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085596800:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787828397.576742:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828397.576749:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828397/real 1787828397] req@ffff9e35da718b40 x1874672085596800/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828422 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828397.576756:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718b40 x1874672085596800/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828422 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828397.576762:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828397.576763:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828397.576766:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085596800:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828397.703889:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828397.703889:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828397.703919:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828398.727920:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828398.727920:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828398.727994:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828399.751904:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828399.751906:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828399.751935:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828400.775895:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828400.775924:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828400.775953:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828401.799920:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828401.799960:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828401.800053:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828402.696015:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828402.696035:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787828402.696042:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828402.696075:0:27606:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1838 00000100:00080000:3.0:1787828402.696095:0:27606:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828402.696107:0:27606:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787828402.696169:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499a40 x1874672085596928/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828402.696226:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828402.696230:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828402.696233:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828402.696237:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1838 00000100:00080000:3.0:1787828402.696242:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828402.696246:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828402.696261:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828402.696276:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649ad00 x1874672085597056/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828402.696316:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787828402.696319:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085596928:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1787828402.696334:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787828402.696338:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828402.696344:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1838 00000100:00080000:3.0:1787828402.696348:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828402.696352:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828402.696358:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828402.696396:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b0c0 x1874672085597184/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828402.696410:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828402.696414:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787828402.696416:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828402.696422:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1838 00000100:00080000:3.0:1787828402.696427:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828402.696431:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828402.696447:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000800:00000100:1.0:1787828402.696471:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828402.696472:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b480 x1874672085597312/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828402.696492:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828402.696499:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787828402.696503:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828402.696510:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1838 00000100:00080000:3.0:1787828402.696518:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828402.696526:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787828402.696535:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828402/real 1787828402] req@ffff9e35c6499a40 x1874672085596928/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828427 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1787828402.696537:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:3.0:1787828402.696564:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499e00 x1874672085597440/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828402.696570:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499a40 x1874672085596928/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828427 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828402.696600:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828402.696603:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828402.696608:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:3.0:1787828402.696611:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:1.0:1787828402.696618:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085596928:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1787828402.696619:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828402.696633:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1838 00000100:00080000:3.0:1787828402.696639:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828402.696644:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828402.696651:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828402.696665:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085597056:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787828402.696667:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c64983c0 x1874672085597568/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828402.696689:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828402.696694:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787828402.696696:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828402.696701:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1838 00000800:00000100:1.0:1787828402.696726:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1787828402.696734:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828402.696742:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828402.696749:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828402.696752:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828402/real 1787828402] req@ffff9e35c649ad00 x1874672085597056/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828427 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828402.696767:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a1c0 x1874672085597696/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828402.696797:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828402.696801:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787828402.696804:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828402.696810:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1838 00000100:00100000:1.0:1787828402.696847:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649ad00 x1874672085597056/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828427 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828402.696866:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828402.696884:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828402.696885:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:1.0:1787828402.696902:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000008:00100000:3.0:1787828402.696903:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:1.0:1787828402.696913:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085597056:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828402.696925:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085597184:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787828402.696934:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498000 x1874672085597824/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828402.696955:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1787828402.696963:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1787828402.696967:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787828402.696972:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828402.696981:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1838 00000100:00100000:1.0:1787828402.696987:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828402/real 1787828402] req@ffff9e35c649b0c0 x1874672085597184/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828427 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828402.696990:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828402.697002:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828402.697014:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:1.0:1787828402.697030:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b0c0 x1874672085597184/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828427 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828402.697040:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c64992c0 x1874672085597952/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828402.697044:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828402.697058:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828402.697068:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085597184:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828402.697077:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085597312:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828402.697102:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828402.697126:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828402/real 1787828402] req@ffff9e35c649b480 x1874672085597312/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828427 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828402.697140:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b480 x1874672085597312/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828427 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828402.697149:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828402.697153:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828402.697157:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085597312:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828402.697163:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085597440:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828402.697183:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828402.697193:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828402/real 1787828402] req@ffff9e35c6499e00 x1874672085597440/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828427 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828402.697222:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499e00 x1874672085597440/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828427 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828402.697238:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828402.697241:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828402.697244:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085597440:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828402.697265:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c64983c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085597568:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787828402.697302:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828402.697312:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828402/real 1787828402] req@ffff9e35c64983c0 x1874672085597568/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828427 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828402.697324:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c64983c0 x1874672085597568/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828427 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828402.697332:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828402.697335:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828402.697338:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c64983c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085597568:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828402.697344:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085597696:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787828402.697359:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828402.697368:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828402/real 1787828402] req@ffff9e35c649a1c0 x1874672085597696/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828427 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828402.697403:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a1c0 x1874672085597696/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828427 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828402.697412:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828402.697415:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828402.697419:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085597696:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828402.697426:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085597824:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828402.697448:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828402.697458:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828402/real 1787828402] req@ffff9e35c6498000 x1874672085597824/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828427 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828402.697469:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498000 x1874672085597824/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828427 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828402.697478:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828402.697481:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828402.697483:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085597824:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828402.697488:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c64992c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085597952:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828402.697503:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828402.697512:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828402/real 1787828402] req@ffff9e35c64992c0 x1874672085597952/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828427 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828402.697524:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c64992c0 x1874672085597952/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828427 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828402.697531:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828402.697534:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828402.697537:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c64992c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085597952:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828402.823992:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828402.824053:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828402.824130:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828403.847976:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828403.847997:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:1.0:1787828403.848048:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828404.871909:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828404.872023:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828404.872131:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787828405.896035:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828405.896192:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828406.919880:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828406.919906:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787828406.919938:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828407.815868:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828407.815876:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787828407.815880:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828407.815888:0:27606:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1843 00000100:00080000:3.0:1787828407.815897:0:27606:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828407.815903:0:27606:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787828407.815935:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c64992c0 x1874672085598080/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828407.815958:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828407.815961:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828407.815962:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828407.815972:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1843 00000100:00080000:3.0:1787828407.815974:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828407.815978:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828407.815986:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828407.815997:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498000 x1874672085598208/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828407.816005:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828407.816009:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:1.0:1787828407.816010:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c64992c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085598080:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1787828407.816010:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828407.816018:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1843 00000100:00080000:3.0:1787828407.816022:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828407.816024:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828407.816027:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828407.816036:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a1c0 x1874672085598336/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828407.816042:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828407.816045:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787828407.816046:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828407.816050:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1843 00000100:00080000:3.0:1787828407.816053:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828407.816056:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828407.816059:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:3.0:1787828407.816066:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c64983c0 x1874672085598464/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787828407.816074:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1787828407.816076:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828407.816081:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787828407.816084:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828407.816090:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1843 00000100:00080000:3.0:1787828407.816095:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828407.816099:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828407.816105:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:1.0:1787828407.816114:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828407/real 1787828407] req@ffff9e35c64992c0 x1874672085598080/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828432 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828407.816122:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499e00 x1874672085598592/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828407.816133:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c64992c0 x1874672085598080/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828432 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828407.816137:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828407.816144:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828407.816147:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828407.816152:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828407.816153:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1843 00000100:00080000:1.0:1787828407.816160:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:3.0:1787828407.816160:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828407.816168:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787828407.816169:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c64992c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085598080:192.168.203.141@tcp:250:kworker.0 00000008:00100000:3.0:1787828407.816174:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828407.816186:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b480 x1874672085598720/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828407.816193:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085598208:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1787828407.816197:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828407.816200:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787828407.816202:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828407.816205:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1843 00000100:00080000:3.0:1787828407.816209:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828407.816211:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:1.0:1787828407.816215:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:3.0:1787828407.816217:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828407.816229:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b0c0 x1874672085598848/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828407.816230:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828407/real 1787828407] req@ffff9e35c6498000 x1874672085598208/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828432 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828407.816252:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828407.816255:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:1.0:1787828407.816256:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498000 x1874672085598208/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828432 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828407.816258:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828407.816267:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1843 00000100:00080000:1.0:1787828407.816270:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828407.816271:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828407.816277:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:1.0:1787828407.816279:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000008:00100000:3.0:1787828407.816282:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:1.0:1787828407.816284:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085598208:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787828407.816293:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649ad00 x1874672085598976/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828407.816294:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085598336:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1787828407.816313:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828407.816316:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787828407.816318:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828407.816322:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1843 00000800:00000100:1.0:1787828407.816325:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1787828407.816327:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828407.816333:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787828407.816350:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828407/real 1787828407] req@ffff9e35c649a1c0 x1874672085598336/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828432 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1787828407.816352:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:3.0:1787828407.816368:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499a40 x1874672085599104/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828407.816369:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a1c0 x1874672085598336/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828432 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828407.816397:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828407.816401:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828407.816404:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085598336:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828407.816409:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c64983c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085598464:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828407.816427:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828407.816434:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828407/real 1787828407] req@ffff9e35c64983c0 x1874672085598464/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828432 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828407.816441:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c64983c0 x1874672085598464/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828432 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828407.816446:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828407.816448:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828407.816450:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c64983c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085598464:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828407.816453:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085598592:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828407.816463:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828407.816469:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828407/real 1787828407] req@ffff9e35c6499e00 x1874672085598592/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828432 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828407.816481:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499e00 x1874672085598592/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828432 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828407.816487:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828407.816488:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828407.816490:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085598592:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828407.816502:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085598720:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787828407.816512:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828407.816519:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828407/real 1787828407] req@ffff9e35c649b480 x1874672085598720/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828432 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828407.816526:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b480 x1874672085598720/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828432 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828407.816531:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828407.816533:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828407.816535:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085598720:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828407.816538:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085598848:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787828407.816548:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828407.816554:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828407/real 1787828407] req@ffff9e35c649b0c0 x1874672085598848/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828432 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828407.816561:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b0c0 x1874672085598848/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828432 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828407.816570:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828407.816572:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828407.816574:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085598848:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828407.816577:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085598976:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828407.816586:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828407.816591:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828407/real 1787828407] req@ffff9e35c649ad00 x1874672085598976/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828432 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828407.816598:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649ad00 x1874672085598976/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828432 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828407.816603:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828407.816605:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828407.816607:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085598976:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828407.816610:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085599104:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828407.816619:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828407.816625:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828407/real 1787828407] req@ffff9e35c6499a40 x1874672085599104/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828432 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828407.816632:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499a40 x1874672085599104/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828432 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828407.816637:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828407.816638:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828407.816641:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085599104:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828407.943874:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828407.943993:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828407.944058:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828408.967862:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828408.967919:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828408.967937:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787828408.967967:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828409.991928:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828409.991933:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787828409.991964:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828409.991965:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828411.015953:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828411.015954:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828411.016066:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828412.039966:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828412.040022:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00080000:3.0:1787828412.935902:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828412.935913:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787828412.935917:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828412.935926:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1848 00000100:00080000:3.0:1787828412.935938:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828412.935945:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787828412.935983:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a580 x1874672085599232/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828412.936014:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828412.936018:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828412.936020:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828412.936023:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1848 00000100:00080000:3.0:1787828412.936026:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828412.936029:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828412.936039:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828412.936051:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499680 x1874672085599360/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828412.936089:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787828412.936095:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085599232:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1787828412.936097:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787828412.936106:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828412.936119:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1848 00000100:00080000:3.0:1787828412.936127:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828412.936135:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828412.936146:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828412.936171:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498780 x1874672085599488/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828412.936192:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1787828412.936194:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1787828412.936204:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787828412.936225:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828412.936231:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1848 00000100:00080000:3.0:1787828412.936238:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828412.936242:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828412.936248:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:1.0:1787828412.936265:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828412/real 1787828412] req@ffff9e35c649a580 x1874672085599232/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828437 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828412.936267:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498b40 x1874672085599616/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828412.936325:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787828412.936331:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a580 x1874672085599232/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828437 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828412.936332:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787828412.936355:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828412.936370:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1848 00000100:00080000:3.0:1787828412.936382:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828412.936391:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828412.936392:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828412.936410:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00080000:1.0:1787828412.936413:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828412.936426:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085599232:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1787828412.936433:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a940 x1874672085599744/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828412.936451:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828412.936456:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828412.936458:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828412.936464:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1848 00000100:00080000:3.0:1787828412.936467:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828412.936472:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828412.936480:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828412.936483:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085599360:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787828412.936498:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649bc00 x1874672085599872/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828412.936514:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828412.936518:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787828412.936521:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828412.936525:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1848 00000100:00080000:3.0:1787828412.936528:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828412.936532:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828412.936537:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828412.936571:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b840 x1874672085600000/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787828412.936586:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1787828412.936603:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828412.936624:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787828412.936627:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787828412.936632:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828412/real 1787828412] req@ffff9e35c6499680 x1874672085599360/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828437 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828412.936633:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1848 00000100:00080000:3.0:1787828412.936640:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828412.936649:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828412.936658:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:1.0:1787828412.936662:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499680 x1874672085599360/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828437 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828412.936674:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1787828412.936675:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498f00 x1874672085600128/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828412.936683:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828412.936695:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085599360:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1787828412.936698:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828412.936704:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:1.0:1787828412.936708:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085599488:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1787828412.936709:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828412.936721:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1848 00000100:00080000:3.0:1787828412.936726:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828412.936730:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828412.936734:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000800:00000100:1.0:1787828412.936744:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828412.936750:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a580 x1874672085600256/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828412.936759:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828412/real 1787828412] req@ffff9e35c6498780 x1874672085599488/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828437 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828412.936774:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498780 x1874672085599488/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828437 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828412.936783:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828412.936787:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828412.936791:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085599488:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828412.936799:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085599616:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828412.936815:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828412.936823:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828412/real 1787828412] req@ffff9e35c6498b40 x1874672085599616/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828437 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828412.936901:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498b40 x1874672085599616/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828437 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828412.936912:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828412.936916:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828412.936920:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085599616:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828412.936928:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085599744:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828412.936963:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828412.936985:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828412/real 1787828412] req@ffff9e35c649a940 x1874672085599744/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828437 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828412.937009:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a940 x1874672085599744/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828437 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828412.937026:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828412.937033:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828412.937039:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085599744:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828412.937067:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085599872:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787828412.937096:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828412.937110:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828412/real 1787828412] req@ffff9e35c649bc00 x1874672085599872/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828437 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828412.937125:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649bc00 x1874672085599872/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828437 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828412.937135:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828412.937139:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828412.937142:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085599872:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828412.937148:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085600000:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787828412.937181:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828412.937190:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828412/real 1787828412] req@ffff9e35c649b840 x1874672085600000/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828437 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828412.937200:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b840 x1874672085600000/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828437 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828412.937208:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828412.937211:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828412.937214:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085600000:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828412.937219:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085600128:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828412.937253:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828412.937263:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828412/real 1787828412] req@ffff9e35c6498f00 x1874672085600128/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828437 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828412.937275:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498f00 x1874672085600128/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828437 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828412.937285:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828412.937288:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828412.937292:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085600128:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828412.937298:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085600256:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828412.937366:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828412.937382:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828412/real 1787828412] req@ffff9e35c649a580 x1874672085600256/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828437 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828412.937422:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a580 x1874672085600256/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828437 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828412.937441:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828412.937447:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828412.937455:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085600256:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828413.063965:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828413.064007:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:1.0:1787828413.064085:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828414.087899:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828414.087919:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828414.088029:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828415.111938:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828415.111978:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828416.135918:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828416.135925:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787828416.136010:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828417.159890:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828417.159956:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828417.160191:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828417.160253:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828418.056326:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085600384:192.168.203.141@tcp:250:kworker.0 00000800:00000100:1.0:1787828418.056430:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828418.056499:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828418/real 1787828418] req@ffff9e35da7192c0 x1874672085600384/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828443 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828418.056600:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7192c0 x1874672085600384/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828443 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828418.056629:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828418.056646:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828418.056660:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085600384:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1787828418.056682:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085600512:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787828418.056739:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828418.056767:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828418/real 1787828418] req@ffff9e35da718b40 x1874672085600512/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828443 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828418.056803:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718b40 x1874672085600512/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828443 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828418.056818:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828418.056823:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828418.056865:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085600512:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828418.056896:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085600640:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787828418.056920:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828418.056933:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828418/real 1787828418] req@ffff9e35da718f00 x1874672085600640/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828443 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828418.056946:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672085600640/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828443 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828418.056955:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828418.056959:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828418.056962:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085600640:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828418.056968:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085600768:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828418.056984:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828418.056994:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828418/real 1787828418] req@ffff9e35da71a940 x1874672085600768/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828443 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828418.057005:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672085600768/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828443 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828418.057014:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828418.057017:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828418.057020:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085600768:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828418.057026:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085600896:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828418.057042:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828418.057052:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828418/real 1787828418] req@ffff9e35da71a1c0 x1874672085600896/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828443 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828418.057064:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a1c0 x1874672085600896/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828443 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828418.057072:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828418.057075:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828418.057078:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085600896:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828418.057084:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085601024:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787828418.057116:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828418.057124:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828418/real 1787828418] req@ffff9e35da7183c0 x1874672085601024/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828443 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828418.057134:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672085601024/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828443 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828418.057141:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828418.057143:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828418.057146:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085601024:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828418.057150:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085601152:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787828418.057183:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828418.057193:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828418/real 1787828418] req@ffff9e35da71a580 x1874672085601152/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828443 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828418.057213:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672085601152/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828443 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828418.057222:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828418.057225:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828418.057228:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085601152:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828418.057233:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085601280:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828418.057249:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828418.057259:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828418/real 1787828418] req@ffff9e35da719a40 x1874672085601280/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828443 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828418.057296:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672085601280/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828443 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828418.057306:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828418.057309:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828418.057312:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085601280:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828418.057319:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085601408:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828418.057341:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828418.057353:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828418/real 1787828418] req@ffff9e35da718780 x1874672085601408/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828443 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828418.057364:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718780 x1874672085601408/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828443 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828418.057380:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828418.057384:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828418.057387:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085601408:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828418.183962:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828418.183972:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828418.184022:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828419.207952:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828419.207971:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828419.208026:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828420.231984:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828420.232000:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828420.232057:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828421.255957:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828421.255957:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828422.279972:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828422.279973:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00080000:2.0:1787828423.175959:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828423.175979:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828423.175986:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828423.176005:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1858 00000100:00080000:2.0:1787828423.176025:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828423.176041:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787828423.176129:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f840 x1874672085601536/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828423.176180:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828423.176188:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828423.176193:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828423.176206:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1858 00000100:00080000:2.0:1787828423.176215:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828423.176225:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828423.176270:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828423.176303:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f0c0 x1874672085601664/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828423.176324:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828423.176331:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787828423.176335:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828423.176343:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1858 00000100:00080000:2.0:1787828423.176350:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828423.176358:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828423.176367:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828423.176387:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f480 x1874672085601792/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828423.176407:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828423.176413:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787828423.176416:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828423.176423:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1858 00000100:00080000:2.0:1787828423.176428:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828423.176435:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828423.176444:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:2.0:1787828423.176463:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e1c0 x1874672085601920/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828423.176484:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828423.176492:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787828423.176496:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828423.176503:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1858 00000100:00080000:2.0:1787828423.176510:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828423.176518:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828423.176526:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:2.0:1787828423.176547:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cb40 x1874672085602048/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828423.176566:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828423.176573:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828423.176577:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828423.176587:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1858 00000100:00080000:2.0:1787828423.176596:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828423.176604:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828423.176615:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828423.176639:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c000 x1874672085602176/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828423.176660:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828423.176667:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787828423.176672:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828423.176682:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1858 00000100:00080000:2.0:1787828423.176692:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828423.176703:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828423.176713:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828423.176734:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e580 x1874672085602304/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828423.176758:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828423.176766:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787828423.176770:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828423.176779:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1858 00000100:00080000:2.0:1787828423.176788:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828423.176855:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828423.176869:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:2.0:1787828423.176898:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cf00 x1874672085602432/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828423.176932:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828423.176940:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787828423.176945:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828423.176956:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1858 00000100:00080000:2.0:1787828423.176964:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828423.176971:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828423.176981:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:2.0:1787828423.176999:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c780 x1874672085602560/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828423.303949:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828423.304004:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828423.304069:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828423.304081:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828424.327887:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828424.327952:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828424.327966:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828425.351922:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787828425.351977:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828425.351978:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828426.375935:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828426.376004:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828426.376043:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828427.400001:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828427.400007:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00080000:1.0:1787828428.295980:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828428.295994:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828428.296001:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828428.296015:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1863 00000100:00080000:1.0:1787828428.296029:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828428.296038:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787828428.296089:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5680 x1874672085602688/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828428.296132:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828428.296138:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828428.296144:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828428.296151:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1863 00000100:00080000:1.0:1787828428.296157:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828428.296164:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828428.296178:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828428.296226:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f61c0 x1874672085602816/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828428.296273:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828428.296300:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787828428.296306:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828428.296334:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1863 00000100:00080000:1.0:1787828428.296342:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828428.296350:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828428.296358:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828428.296384:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5a40 x1874672085602944/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828428.296397:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828428.296402:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828428.296405:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828428.296415:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1863 00000100:00080000:1.0:1787828428.296422:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828428.296433:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828428.296449:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:1.0:1787828428.296476:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6580 x1874672085603072/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828428.296513:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828428.296526:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828428.296537:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828428.296576:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1863 00000100:00080000:1.0:1787828428.296588:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828428.296609:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828428.296643:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:1.0:1787828428.296671:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6940 x1874672085603200/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828428.296693:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828428.296713:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828428.296718:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828428.296729:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1863 00000100:00080000:1.0:1787828428.296739:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828428.296760:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828428.296775:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828428.296810:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f70c0 x1874672085603328/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828428.296829:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828428.296835:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787828428.296838:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828428.296847:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1863 00000100:00080000:1.0:1787828428.296853:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828428.296860:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828428.296870:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828428.296951:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5680 x1874672085603456/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828428.296975:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828428.297003:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828428.297010:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828428.297033:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1863 00000100:00080000:1.0:1787828428.297046:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828428.297071:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828428.297089:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:1.0:1787828428.297126:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4000 x1874672085603584/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828428.297153:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828428.297190:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828428.297217:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828428.297252:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1863 00000100:00080000:1.0:1787828428.297270:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828428.297293:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828428.297307:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:1.0:1787828428.297352:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f43c0 x1874672085603712/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828428.423963:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828428.423963:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828428.424083:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828429.447919:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787828429.447924:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828429.447989:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828429.448041:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828430.472015:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828430.472021:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828430.472038:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828431.495972:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828431.496043:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828431.496402:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828432.520041:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828432.520042:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828432.520115:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828433.415967:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828433.415975:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787828433.415978:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828433.415986:0:27606:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1868 00000100:00080000:3.0:1787828433.415995:0:27606:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828433.416001:0:27606:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787828433.416063:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499a40 x1874672085603840/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828433.416095:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828433.416099:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828433.416101:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828433.416105:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1868 00000100:00080000:3.0:1787828433.416108:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828433.416112:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828433.416132:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828433.416144:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649ad00 x1874672085603968/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828433.416154:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828433.416158:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787828433.416193:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828433.416197:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1868 00000100:00080000:3.0:1787828433.416218:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828433.416221:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828433.416225:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828433.416233:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b0c0 x1874672085604096/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828433.416248:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828433.416257:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787828433.416259:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828433.416262:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1868 00000100:00080000:3.0:1787828433.416264:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828433.416266:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828433.416270:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:3.0:1787828433.416285:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b480 x1874672085604224/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828433.416290:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828433.416293:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787828433.416294:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828433.416297:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1868 00000100:00080000:3.0:1787828433.416299:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828433.416302:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828433.416305:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:3.0:1787828433.416312:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499e00 x1874672085604352/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828433.416318:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828433.416321:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828433.416322:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828433.416324:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1868 00000100:00080000:3.0:1787828433.416327:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828433.416329:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828433.416333:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828433.416348:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c64983c0 x1874672085604480/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828433.416356:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828433.416359:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787828433.416361:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828433.416365:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1868 00000100:00080000:3.0:1787828433.416369:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828433.416372:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828433.416375:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828433.416383:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a1c0 x1874672085604608/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828433.416390:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828433.416392:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787828433.416394:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828433.416404:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1868 00000100:00080000:3.0:1787828433.416412:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828433.416426:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828433.416432:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:3.0:1787828433.416456:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498000 x1874672085604736/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828433.416466:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828433.416486:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787828433.416493:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828433.416516:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1868 00000100:00080000:3.0:1787828433.416523:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828433.416530:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828433.416551:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:3.0:1787828433.416568:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c64992c0 x1874672085604864/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828433.543905:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828433.543944:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828433.544018:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828434.567933:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828434.567985:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828434.568006:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828435.591951:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828435.592015:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787828435.592072:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828436.615999:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828436.616000:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828436.616062:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828437.639919:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828437.639992:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828437.640102:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828438.535988:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828438.536010:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828438.536021:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828438.536040:0:27606:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1873 00000100:00080000:2.0:1787828438.536058:0:27606:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828438.536069:0:27606:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787828438.536157:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e940 x1874672085604992/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828438.536204:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828438.536211:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828438.536215:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828438.536224:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1873 00000100:00080000:2.0:1787828438.536231:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828438.536242:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828438.536265:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828438.536299:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498d680 x1874672085605120/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828438.536350:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828438.536357:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787828438.536361:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828438.536371:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1873 00000100:00080000:2.0:1787828438.536378:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828438.536385:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828438.536395:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828438.536418:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498fc00 x1874672085605248/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828438.536463:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828438.536469:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787828438.536472:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828438.536478:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1873 00000100:00080000:2.0:1787828438.536485:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828438.536490:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828438.536497:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:2.0:1787828438.536513:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498de00 x1874672085605376/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828438.536526:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828438.536531:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787828438.536538:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828438.536550:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1873 00000100:00080000:2.0:1787828438.536566:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828438.536589:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828438.536602:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:2.0:1787828438.536629:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498ed00 x1874672085605504/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828438.536723:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828438.536741:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828438.536748:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828438.536767:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1873 00000100:00080000:2.0:1787828438.536786:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828438.536871:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828438.536883:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828438.536907:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c3c0 x1874672085605632/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828438.536929:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828438.536937:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787828438.536941:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828438.536951:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1873 00000100:00080000:2.0:1787828438.536962:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828438.536971:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828438.536984:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828438.537024:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e940 x1874672085605760/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828438.537078:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828438.537084:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787828438.537089:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828438.537100:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1873 00000100:00080000:2.0:1787828438.537108:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828438.537120:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828438.537168:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:2.0:1787828438.537213:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c780 x1874672085605888/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828438.537367:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828438.537387:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787828438.537396:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828438.537412:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1873 00000100:00080000:2.0:1787828438.537422:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828438.537432:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828438.537443:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:2.0:1787828438.537469:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cf00 x1874672085606016/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828438.663942:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828438.663977:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787828438.664060:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828439.687987:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828439.688013:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828439.688060:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828440.712039:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787828440.712045:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828440.712206:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828441.735966:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828441.736053:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828441.736053:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828442.759973:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828442.759974:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828442.760001:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828443.655886:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828443.655899:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828443.655903:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828443.655932:0:27606:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1879 00000100:00080000:2.0:1787828443.655943:0:27606:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828443.655950:0:27606:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787828443.656003:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e580 x1874672085606144/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828443.656029:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828443.656033:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828443.656035:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828443.656039:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1879 00000100:00080000:2.0:1787828443.656042:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828443.656046:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828443.656056:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828443.656069:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c000 x1874672085606272/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828443.656083:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828443.656101:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787828443.656104:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828443.656108:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1879 00000100:00080000:2.0:1787828443.656112:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828443.656115:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828443.656121:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828443.656135:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cb40 x1874672085606400/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828443.656145:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828443.656147:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787828443.656149:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828443.656153:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1879 00000100:00080000:2.0:1787828443.656156:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828443.656159:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828443.656165:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:2.0:1787828443.656181:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e1c0 x1874672085606528/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828443.656191:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828443.656195:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787828443.656197:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828443.656201:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1879 00000100:00080000:2.0:1787828443.656205:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828443.656213:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828443.656222:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:2.0:1787828443.656241:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f480 x1874672085606656/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828443.656277:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828443.656282:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828443.656286:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828443.656297:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1879 00000100:00080000:2.0:1787828443.656303:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828443.656310:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828443.656314:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828443.656323:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f0c0 x1874672085606784/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828443.656341:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828443.656345:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787828443.656347:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828443.656353:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1879 00000100:00080000:2.0:1787828443.656357:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828443.656360:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828443.656365:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828443.656379:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f840 x1874672085606912/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828443.656390:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828443.656395:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787828443.656401:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828443.656414:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1879 00000100:00080000:2.0:1787828443.656424:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828443.656438:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828443.656451:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:2.0:1787828443.656476:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e580 x1874672085607040/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828443.656487:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828443.656490:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787828443.656493:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828443.656497:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1879 00000100:00080000:2.0:1787828443.656502:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828443.656511:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828443.656519:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:2.0:1787828443.656534:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cf00 x1874672085607168/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828443.783983:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828443.783996:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828443.784077:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828444.807871:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787828444.807904:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828444.807931:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828445.831892:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828445.831956:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828445.831961:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828446.855881:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828446.855888:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828446.855917:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828447.879897:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828447.879898:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828447.879909:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828447.879927:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787828448.776024:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085607296:192.168.203.141@tcp:250:kworker.0 00000800:00000100:1.0:1787828448.776111:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828448.776155:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828448/real 1787828448] req@ffff9e35da718000 x1874672085607296/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828473 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828448.776191:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718000 x1874672085607296/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828473 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828448.776220:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828448.776230:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828448.776242:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085607296:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1787828448.776278:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085607424:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787828448.776319:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828448.776334:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828448/real 1787828448] req@ffff9e35da718780 x1874672085607424/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828473 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828448.776355:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718780 x1874672085607424/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828473 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828448.776369:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828448.776385:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828448.776389:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085607424:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828448.776398:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085607552:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787828448.776423:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828448.776432:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828448/real 1787828448] req@ffff9e35da719a40 x1874672085607552/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828473 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828448.776440:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672085607552/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828473 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828448.776445:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828448.776447:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828448.776449:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085607552:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828448.776454:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085607680:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828448.776471:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828448.776477:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828448/real 1787828448] req@ffff9e35da71a580 x1874672085607680/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828473 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828448.776485:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672085607680/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828473 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828448.776490:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828448.776492:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828448.776494:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085607680:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828448.776497:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085607808:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828448.776507:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828448.776524:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828448/real 1787828448] req@ffff9e35da7183c0 x1874672085607808/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828473 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828448.776532:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672085607808/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828473 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828448.776537:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828448.776539:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828448.776541:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085607808:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828448.776545:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085607936:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787828448.776554:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828448.776560:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828448/real 1787828448] req@ffff9e35da71a1c0 x1874672085607936/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828473 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828448.776568:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a1c0 x1874672085607936/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828473 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828448.776573:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828448.776575:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828448.776577:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085607936:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828448.776580:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085608064:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787828448.776590:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828448.776595:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828448/real 1787828448] req@ffff9e35da71a940 x1874672085608064/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828473 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828448.776602:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672085608064/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828473 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828448.776607:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828448.776609:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828448.776611:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085608064:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828448.776624:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085608192:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828448.776634:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828448.776641:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828448/real 1787828448] req@ffff9e35da718f00 x1874672085608192/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828473 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828448.776648:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672085608192/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828473 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828448.776658:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828448.776660:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828448.776662:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085608192:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828448.776665:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085608320:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828448.776681:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828448.776687:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828448/real 1787828448] req@ffff9e35da718b40 x1874672085608320/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828473 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828448.776694:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718b40 x1874672085608320/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828473 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828448.776699:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828448.776701:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828448.776703:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085608320:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828448.903940:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828448.903940:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787828448.904009:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828449.927867:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787828449.927893:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828449.927968:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828450.951946:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828450.952033:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828450.952084:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828451.975903:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828451.975996:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828451.976029:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828452.999945:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828452.999980:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828452.999981:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828453.895973:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828453.895986:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828453.895992:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828453.896004:0:27606:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1889 00000100:00080000:1.0:1787828453.896044:0:27606:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828453.896056:0:27606:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787828453.896106:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4780 x1874672085608448/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828453.896148:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828453.896153:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828453.896155:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828453.896162:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1889 00000100:00080000:1.0:1787828453.896170:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828453.896179:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828453.896197:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828453.896229:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7840 x1874672085608576/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828453.896257:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828453.896264:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787828453.896268:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828453.896274:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1889 00000100:00080000:1.0:1787828453.896297:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828453.896303:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828453.896311:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828453.896330:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4f00 x1874672085608704/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828453.896344:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828453.896348:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828453.896352:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828453.896360:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1889 00000100:00080000:1.0:1787828453.896367:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828453.896372:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828453.896380:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:1.0:1787828453.896399:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5e00 x1874672085608832/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828453.896422:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828453.896426:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828453.896430:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828453.896435:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1889 00000100:00080000:1.0:1787828453.896444:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828453.896450:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828453.896457:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:1.0:1787828453.896477:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7480 x1874672085608960/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828453.896492:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828453.896497:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828453.896500:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828453.896506:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1889 00000100:00080000:1.0:1787828453.896512:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828453.896516:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828453.896524:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828453.896549:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6d00 x1874672085609088/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828453.896603:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828453.896612:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787828453.896617:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828453.896627:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1889 00000100:00080000:1.0:1787828453.896637:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828453.896656:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828453.896665:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828453.896692:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4b40 x1874672085609216/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828453.896707:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828453.896712:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828453.896715:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828453.896722:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1889 00000100:00080000:1.0:1787828453.896729:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828453.896750:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828453.896760:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:1.0:1787828453.896780:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f43c0 x1874672085609344/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828453.896794:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828453.896845:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828453.896848:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828453.896856:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1889 00000100:00080000:1.0:1787828453.896864:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828453.896879:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828453.896907:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:1.0:1787828453.896934:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4000 x1874672085609472/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828454.024012:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828454.024065:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828454.024104:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828455.047980:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828455.047995:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828455.048055:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828455.048091:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828456.071969:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828456.071973:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828456.072059:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828456.072069:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000400:00000100:2.0:1787828456.968334:0:2396:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.141:988 00000400:00000100:2.0:1787828456.968431:0:2396:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.141@tcp at host 192.168.203.141 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1787828456.968456:0:2396:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:2.0:1787828456.968536:0:2396:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000800:00000100:2.0:1787828456.968566:0:2396:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:2.0:1787828456.968581:0:2396:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:2.0:1787828456.968597:0:2396:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:2.0:1787828456.968616:0:2396:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:2.0:1787828456.968644:0:2396:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:2.0:1787828456.968667:0:2396:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:2.0:1787828456.968694:0:2396:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:2.0:1787828457.095962:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828457.095976:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828457.096015:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828457.096060:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828458.119975:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828458.120031:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828458.120055:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828459.016365:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828459.016378:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828459.016384:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828459.016417:0:27606:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1894 00000100:00080000:2.0:1787828459.016452:0:27606:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828459.016496:0:27606:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787828459.016551:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c780 x1874672085609600/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828459.016621:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828459.016631:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828459.016635:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828459.016646:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1894 00000100:00080000:2.0:1787828459.016657:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828459.016667:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828459.016685:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828459.016718:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e940 x1874672085609728/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828459.016742:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828459.016749:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787828459.016753:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828459.016760:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1894 00000100:00080000:2.0:1787828459.016766:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828459.016774:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828459.016782:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828459.016857:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c3c0 x1874672085609856/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828459.016880:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828459.016887:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787828459.016891:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828459.016900:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1894 00000100:00080000:2.0:1787828459.016909:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828459.016917:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828459.016928:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:2.0:1787828459.016951:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498ed00 x1874672085609984/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828459.016967:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828459.016974:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787828459.016977:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828459.017013:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1894 00000100:00080000:2.0:1787828459.017020:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828459.017026:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828459.017035:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:2.0:1787828459.017056:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498de00 x1874672085610112/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828459.017075:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828459.017080:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828459.017084:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828459.017090:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1894 00000100:00080000:2.0:1787828459.017097:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828459.017103:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828459.017111:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828459.017131:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498fc00 x1874672085610240/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828459.017150:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828459.017156:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787828459.017160:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828459.017167:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1894 00000100:00080000:2.0:1787828459.017173:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828459.017180:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828459.017189:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828459.017209:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498d680 x1874672085610368/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828459.017226:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828459.017231:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787828459.017234:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828459.017241:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1894 00000100:00080000:2.0:1787828459.017248:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828459.017255:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828459.017264:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:2.0:1787828459.017284:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cf00 x1874672085610496/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828459.017316:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828459.017323:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787828459.017327:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828459.017334:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1894 00000100:00080000:2.0:1787828459.017341:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828459.017348:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828459.017356:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:2.0:1787828459.017450:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e580 x1874672085610624/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828459.017665:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085609600:192.168.203.141@tcp:250:kworker.0 00000800:00000100:1.0:1787828459.017764:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828459.017870:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828459/real 1787828459] req@ffff9e35c498c780 x1874672085609600/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828484 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828459.017907:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c780 x1874672085609600/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828484 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828459.017927:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828459.017936:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828459.017946:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085609600:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1787828459.017978:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085609728:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787828459.018047:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828459.018080:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828459/real 1787828459] req@ffff9e35c498e940 x1874672085609728/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828484 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828459.018106:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e940 x1874672085609728/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828484 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828459.018123:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828459.018131:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828459.018140:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085609728:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828459.018151:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085609856:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787828459.018188:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828459.018213:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828459/real 1787828459] req@ffff9e35c498c3c0 x1874672085609856/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828484 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828459.018232:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c3c0 x1874672085609856/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828484 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828459.018251:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828459.018257:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828459.018262:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085609856:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828459.018272:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085609984:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828459.018300:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828459.018319:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828459/real 1787828459] req@ffff9e35c498ed00 x1874672085609984/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828484 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828459.018356:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498ed00 x1874672085609984/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828484 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828459.018372:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828459.018377:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828459.018383:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085609984:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828459.018393:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085610112:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828459.018436:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828459.018452:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828459/real 1787828459] req@ffff9e35c498de00 x1874672085610112/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828484 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828459.018470:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498de00 x1874672085610112/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828484 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828459.018484:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828459.018488:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828459.018494:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085610112:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828459.018504:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085610240:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787828459.018533:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828459.018563:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828459/real 1787828459] req@ffff9e35c498fc00 x1874672085610240/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828484 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828459.018590:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498fc00 x1874672085610240/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828484 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828459.018605:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828459.018612:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828459.018621:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085610240:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828459.018634:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085610368:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787828459.018677:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828459.018705:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828459/real 1787828459] req@ffff9e35c498d680 x1874672085610368/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828484 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828459.018737:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498d680 x1874672085610368/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828484 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828459.018755:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828459.018761:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828459.018768:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085610368:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828459.018779:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085610496:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828459.018864:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828459.018892:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828459/real 1787828459] req@ffff9e35c498cf00 x1874672085610496/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828484 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828459.018916:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498cf00 x1874672085610496/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828484 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828459.018930:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828459.018936:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828459.018941:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085610496:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828459.018955:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085610624:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828459.019008:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828459.019032:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828459/real 1787828459] req@ffff9e35c498e580 x1874672085610624/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828484 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828459.019051:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e580 x1874672085610624/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828484 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828459.019064:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828459.019069:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828459.019074:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085610624:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828459.143932:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828459.143932:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828459.144034:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828459.144044:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828460.167925:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828460.167927:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828460.168035:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828461.191901:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828461.191901:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828461.192063:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828462.215904:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828462.216047:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828463.239980:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828463.240022:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828463.240028:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00080000:3.0:1787828464.136063:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828464.136076:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787828464.136081:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828464.136096:0:27606:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1899 00000100:00080000:3.0:1787828464.136111:0:27606:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828464.136121:0:27606:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787828464.136180:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c64992c0 x1874672085610752/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828464.136219:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828464.136224:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828464.136226:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828464.136242:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1899 00000100:00080000:3.0:1787828464.136249:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828464.136257:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828464.136275:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828464.136290:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c64992c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085610752:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1787828464.136301:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498000 x1874672085610880/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828464.136323:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828464.136328:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787828464.136330:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828464.136335:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1899 00000100:00080000:3.0:1787828464.136339:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828464.136344:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828464.136350:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828464.136365:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a1c0 x1874672085611008/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787828464.136376:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1787828464.136403:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828464.136407:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787828464.136410:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828464.136419:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1899 00000100:00100000:1.0:1787828464.136426:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828464/real 1787828464] req@ffff9e35c64992c0 x1874672085610752/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828489 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828464.136426:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828464.136444:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787828464.136456:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c64992c0 x1874672085610752/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828489 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1787828464.136458:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00080000:1.0:1787828464.136482:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828464.136491:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828464.136492:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c64983c0 x1874672085611136/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828464.136517:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c64992c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085610752:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1787828464.136531:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828464.136536:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787828464.136539:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828464.136546:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1899 00000100:00080000:3.0:1787828464.136550:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828464.136556:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787828464.136559:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085610880:192.168.203.141@tcp:38:kworker.0 00000008:00100000:3.0:1787828464.136568:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:3.0:1787828464.136585:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499e00 x1874672085611264/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828464.136597:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1787828464.136606:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1787828464.136611:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828464.136614:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828464.136622:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1899 00000100:00080000:3.0:1787828464.136627:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1787828464.136632:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828464/real 1787828464] req@ffff9e35c6498000 x1874672085610880/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828489 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828464.136638:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828464.136651:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828464.136661:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498000 x1874672085610880/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828489 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828464.136668:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b480 x1874672085611392/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828464.136700:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828464.136711:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828464.136717:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:3.0:1787828464.136719:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787828464.136724:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787828464.136726:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085610880:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1787828464.136733:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1899 00000100:00080000:3.0:1787828464.136741:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1787828464.136743:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085611008:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1787828464.136751:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828464.136775:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1787828464.136782:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828464.136793:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b0c0 x1874672085611520/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828464.136822:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828464/real 1787828464] req@ffff9e35c649a1c0 x1874672085611008/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828489 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828464.136837:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a1c0 x1874672085611008/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828489 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828464.136844:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828464.136847:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828464.136851:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085611008:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828464.136859:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c64983c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085611136:192.168.203.141@tcp:8:kworker.0 00000100:00080000:3.0:1787828464.136862:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828464.136868:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787828464.136872:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828464.136880:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1899 00000800:00000100:1.0:1787828464.136881:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1787828464.136896:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1787828464.136904:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828464/real 1787828464] req@ffff9e35c64983c0 x1874672085611136/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828489 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828464.136908:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828464.136925:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:1.0:1787828464.136979:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c64983c0 x1874672085611136/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828489 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828464.136998:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649ad00 x1874672085611648/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828464.137050:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828464.137059:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828464.137069:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:3.0:1787828464.137070:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787828464.137076:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828464.137088:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1899 00000100:00100000:1.0:1787828464.137094:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c64983c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085611136:192.168.203.141@tcp:8:kworker.0 00000100:00080000:3.0:1787828464.137097:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828464.137115:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828464.137124:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:3.0:1787828464.137142:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499a40 x1874672085611776/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828464.137158:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085611264:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828464.137239:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828464.137250:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828464/real 1787828464] req@ffff9e35c6499e00 x1874672085611264/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828489 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828464.137260:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499e00 x1874672085611264/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828489 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828464.137269:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828464.137271:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828464.137274:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085611264:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828464.137279:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085611392:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787828464.137293:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828464.137300:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828464/real 1787828464] req@ffff9e35c649b480 x1874672085611392/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828489 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828464.137309:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b480 x1874672085611392/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828489 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828464.137320:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828464.137323:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828464.137325:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085611392:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828464.137329:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085611520:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787828464.137340:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828464.137347:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828464/real 1787828464] req@ffff9e35c649b0c0 x1874672085611520/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828489 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828464.137354:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b0c0 x1874672085611520/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828489 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828464.137360:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828464.137363:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828464.137365:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085611520:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828464.137369:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085611648:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828464.137382:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828464.137388:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828464/real 1787828464] req@ffff9e35c649ad00 x1874672085611648/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828489 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828464.137396:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649ad00 x1874672085611648/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828489 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828464.137402:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828464.137404:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828464.137407:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085611648:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828464.137421:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085611776:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828464.137444:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828464.137450:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828464/real 1787828464] req@ffff9e35c6499a40 x1874672085611776/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828489 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828464.137459:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499a40 x1874672085611776/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828489 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828464.137465:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828464.137467:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828464.137469:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085611776:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828464.263911:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787828464.263967:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828465.288017:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828465.288017:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828465.288053:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828466.312003:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828466.312057:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828466.312071:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787828467.336011:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828467.336074:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828468.359948:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828468.359948:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828468.359996:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828469.257198:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085611904:192.168.203.141@tcp:250:kworker.0 00000800:00000100:1.0:1787828469.257257:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828469.257318:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828469/real 1787828469] req@ffff9e35da7192c0 x1874672085611904/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828494 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828469.257351:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7192c0 x1874672085611904/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828494 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828469.257373:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828469.257379:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828469.257385:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085611904:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1787828469.257415:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085612032:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787828469.257443:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828469.257459:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828469/real 1787828469] req@ffff9e35da718b40 x1874672085612032/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828494 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828469.257486:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718b40 x1874672085612032/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828494 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828469.257503:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828469.257506:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828469.257509:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085612032:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828469.257515:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085612160:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787828469.257528:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828469.257535:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828469/real 1787828469] req@ffff9e35da718f00 x1874672085612160/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828494 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828469.257542:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672085612160/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828494 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828469.257548:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828469.257550:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828469.257552:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085612160:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828469.257556:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085612288:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828469.257566:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828469.257571:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828469/real 1787828469] req@ffff9e35da71a940 x1874672085612288/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828494 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828469.257579:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672085612288/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828494 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828469.257585:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828469.257586:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828469.257588:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085612288:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828469.257592:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085612416:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828469.257601:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828469.257607:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828469/real 1787828469] req@ffff9e35da71a1c0 x1874672085612416/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828494 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828469.257614:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a1c0 x1874672085612416/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828494 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828469.257620:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828469.257622:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828469.257624:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085612416:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828469.257627:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085612544:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787828469.257637:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828469.257642:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828469/real 1787828469] req@ffff9e35da7183c0 x1874672085612544/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828494 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828469.257650:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672085612544/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828494 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828469.257666:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828469.257670:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828469.257673:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085612544:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828469.257680:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085612672:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787828469.257700:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828469.257711:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828469/real 1787828469] req@ffff9e35da71a580 x1874672085612672/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828494 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828469.257720:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672085612672/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828494 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828469.257726:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828469.257728:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828469.257730:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085612672:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828469.257745:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085612800:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828469.257758:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828469.257765:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828469/real 1787828469] req@ffff9e35da719a40 x1874672085612800/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828494 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828469.257773:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672085612800/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828494 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828469.257778:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828469.257780:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828469.257782:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085612800:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828469.257786:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085612928:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828469.257822:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828469.257830:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828469/real 1787828469] req@ffff9e35da718780 x1874672085612928/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828494 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828469.257838:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718780 x1874672085612928/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828494 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828469.257843:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828469.257845:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828469.257848:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085612928:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828469.383872:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828469.383879:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828469.383927:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828470.407947:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828470.407954:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787828470.408004:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828471.431912:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828471.431919:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828471.432060:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828472.455924:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828472.455971:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828472.455987:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828473.479977:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828473.479992:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787828473.480099:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828474.375997:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828474.376009:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787828474.376014:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828474.376026:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1909 00000100:00080000:3.0:1787828474.376049:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828474.376057:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787828474.376096:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a580 x1874672085613056/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828474.376164:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828474.376171:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828474.376175:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828474.376200:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1909 00000100:00080000:3.0:1787828474.376208:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828474.376214:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828474.376228:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828474.376243:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085613056:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1787828474.376255:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498f00 x1874672085613184/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828474.376277:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828474.376285:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787828474.376288:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828474.376296:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1909 00000100:00080000:3.0:1787828474.376303:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828474.376310:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828474.376319:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1787828474.376335:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828474.376358:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b840 x1874672085613312/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828474.376371:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828474.376375:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787828474.376378:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828474.376383:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1909 00000100:00080000:3.0:1787828474.376388:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828474.376393:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787828474.376399:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828474/real 1787828474] req@ffff9e35c649a580 x1874672085613056/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828499 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1787828474.376401:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:1.0:1787828474.376426:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a580 x1874672085613056/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828499 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828474.376426:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649bc00 x1874672085613440/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828474.376466:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828474.376472:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828474.376473:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828474.376481:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085613056:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1787828474.376481:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787828474.376487:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828474.376495:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1909 00000100:00080000:3.0:1787828474.376517:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828474.376533:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787828474.376541:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085613184:192.168.203.141@tcp:38:kworker.0 00000008:00100000:3.0:1787828474.376547:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:3.0:1787828474.376575:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a940 x1874672085613568/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787828474.376590:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1787828474.376607:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787828474.376614:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828474/real 1787828474] req@ffff9e35c6498f00 x1874672085613184/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828499 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828474.376647:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828474.376653:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828474.376661:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1909 00000100:00080000:3.0:1787828474.376666:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1787828474.376669:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498f00 x1874672085613184/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828499 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828474.376674:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828474.376683:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00080000:1.0:1787828474.376689:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828474.376716:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828474.376720:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498b40 x1874672085613696/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828474.376731:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085613184:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1787828474.376747:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787828474.376756:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085613312:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1787828474.376758:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787828474.376765:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828474.376777:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1909 00000100:00080000:3.0:1787828474.376784:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828474.376791:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:1.0:1787828474.376805:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:3.0:1787828474.376810:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828474.376834:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498780 x1874672085613824/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828474.376852:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828474.376858:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787828474.376862:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787828474.376869:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828474/real 1787828474] req@ffff9e35c649b840 x1874672085613312/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828499 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828474.376929:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b840 x1874672085613312/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828499 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828474.376947:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828474.376955:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828474.376962:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085613312:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1787828474.376966:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1909 00000100:00080000:3.0:1787828474.376979:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1787828474.376982:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085613440:192.168.203.141@tcp:8:kworker.0 00000100:00080000:3.0:1787828474.376993:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828474.377005:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:3.0:1787828474.377028:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499680 x1874672085613952/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787828474.377041:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1787828474.377056:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828474.377065:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787828474.377069:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787828474.377077:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828474/real 1787828474] req@ffff9e35c649bc00 x1874672085613440/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828499 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828474.377082:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1909 00000100:00080000:3.0:1787828474.377094:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828474.377107:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828474.377117:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:1.0:1787828474.377119:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649bc00 x1874672085613440/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828499 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828474.377143:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1787828474.377147:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a580 x1874672085614080/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828474.377157:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828474.377173:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085613440:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828474.377210:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085613568:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828474.377242:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828474.377256:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828474/real 1787828474] req@ffff9e35c649a940 x1874672085613568/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828499 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828474.377269:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a940 x1874672085613568/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828499 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828474.377279:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828474.377283:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828474.377287:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085613568:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828474.377294:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085613696:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787828474.377311:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828474.377321:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828474/real 1787828474] req@ffff9e35c6498b40 x1874672085613696/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828499 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828474.377341:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498b40 x1874672085613696/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828499 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828474.377367:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828474.377369:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828474.377371:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085613696:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828474.377375:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085613824:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787828474.377388:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828474.377395:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828474/real 1787828474] req@ffff9e35c6498780 x1874672085613824/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828499 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828474.377403:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498780 x1874672085613824/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828499 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828474.377409:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828474.377411:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828474.377413:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085613824:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828474.377417:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085613952:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828474.377427:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828474.377433:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828474/real 1787828474] req@ffff9e35c6499680 x1874672085613952/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828499 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828474.377442:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499680 x1874672085613952/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828499 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828474.377464:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828474.377466:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828474.377468:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085613952:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828474.377471:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085614080:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828474.377481:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828474.377487:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828474/real 1787828474] req@ffff9e35c649a580 x1874672085614080/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828499 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828474.377495:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a580 x1874672085614080/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828499 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828474.377500:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828474.377502:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828474.377504:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085614080:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828474.503952:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828474.503994:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828474.504063:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828475.527953:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828475.528072:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828476.551903:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828476.551934:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828476.551953:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828477.575965:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828477.575968:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828477.576091:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828478.599990:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828478.600015:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828478.600137:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828479.496062:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828479.496077:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828479.496083:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828479.496099:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1914 00000100:00080000:1.0:1787828479.496116:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828479.496127:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787828479.496190:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f70c0 x1874672085614208/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828479.496252:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828479.496259:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828479.496263:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828479.496288:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1914 00000100:00080000:1.0:1787828479.496294:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828479.496314:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828479.496332:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828479.496358:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6940 x1874672085614336/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828479.496388:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828479.496394:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787828479.496397:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828479.496403:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1914 00000100:00080000:1.0:1787828479.496408:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828479.496413:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828479.496420:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828479.496436:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6580 x1874672085614464/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828479.496448:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828479.496452:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828479.496454:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828479.496459:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1914 00000100:00080000:1.0:1787828479.496480:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828479.496487:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828479.496495:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:1.0:1787828479.496524:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5a40 x1874672085614592/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828479.496542:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828479.496549:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828479.496552:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828479.496560:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1914 00000100:00080000:1.0:1787828479.496568:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828479.496575:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828479.496588:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:1.0:1787828479.496618:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f61c0 x1874672085614720/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828479.496641:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828479.496655:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828479.496661:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828479.496675:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1914 00000100:00080000:1.0:1787828479.496690:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828479.496704:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828479.496719:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828479.496747:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4780 x1874672085614848/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828479.496794:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828479.496800:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787828479.496803:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828479.496908:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1914 00000100:00080000:1.0:1787828479.496936:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828479.496953:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828479.496966:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828479.496999:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4f00 x1874672085614976/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828479.497051:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828479.497055:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828479.497061:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828479.497068:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1914 00000100:00080000:1.0:1787828479.497075:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828479.497087:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828479.497100:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:1.0:1787828479.497124:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7840 x1874672085615104/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828479.497146:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828479.497157:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828479.497162:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828479.497175:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1914 00000100:00080000:1.0:1787828479.497181:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828479.497191:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828479.497202:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:1.0:1787828479.497230:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7480 x1874672085615232/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787828479.497344:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6580 x1874672085614464/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828504 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828479.497371:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828479.497377:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828479.497383:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085614464:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787828479.497395:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085614592:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787828479.497433:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828479.497454:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828479/real 1787828479] req@ffff9e35da6f5a40 x1874672085614592/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828504 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828479.497477:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5a40 x1874672085614592/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828504 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828479.497493:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828479.497498:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828479.497505:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085614592:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787828479.497516:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085614720:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787828479.497551:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828479.497571:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828479/real 1787828479] req@ffff9e35da6f61c0 x1874672085614720/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828504 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828479.497610:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f61c0 x1874672085614720/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828504 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828479.497626:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828479.497632:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828479.497640:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085614720:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787828479.497685:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085614848:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787828479.497727:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828479.497751:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828479/real 1787828479] req@ffff9e35da6f4780 x1874672085614848/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828504 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828479.497777:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4780 x1874672085614848/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828504 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828479.497794:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828479.497800:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828479.497807:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085614848:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787828479.497820:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085614976:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787828479.497925:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828479.497955:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828479/real 1787828479] req@ffff9e35da6f4f00 x1874672085614976/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828504 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828479.497996:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4f00 x1874672085614976/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828504 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828479.498013:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828479.498030:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828479.498037:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085614976:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787828479.498049:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085615104:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787828479.498104:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828479.498128:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828479/real 1787828479] req@ffff9e35da6f7840 x1874672085615104/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828504 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828479.498151:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7840 x1874672085615104/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828504 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828479.498168:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828479.498174:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828479.498181:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085615104:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787828479.498193:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085615232:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787828479.498242:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828479.498265:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828479/real 1787828479] req@ffff9e35da6f7480 x1874672085615232/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828504 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828479.498304:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7480 x1874672085615232/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828504 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828479.498325:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828479.498332:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828479.498337:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085615232:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828479.623952:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787828479.623974:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828479.623974:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828479.624073:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828480.647941:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828480.647942:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828480.647996:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828480.648027:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828481.671912:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828481.672010:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787828481.672068:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787828481.672074:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828482.695983:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828482.695984:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828482.696034:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828482.696046:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787828483.719909:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828483.719909:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828483.719945:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828483.719959:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787828484.615952:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828484.615963:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828484.615967:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828484.615979:0:27606:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1919 00000100:00080000:1.0:1787828484.615992:0:27606:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828484.616001:0:27606:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787828484.616066:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5e00 x1874672085615360/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828484.616105:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828484.616109:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828484.616111:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828484.616115:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1919 00000100:00080000:1.0:1787828484.616119:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828484.616123:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828484.616136:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828484.616154:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6d00 x1874672085615488/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787828484.616167:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085615360:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787828484.616171:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828484.616197:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787828484.616201:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828484.616207:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1919 00000100:00080000:1.0:1787828484.616212:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828484.616217:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828484.616223:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828484.616240:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4b40 x1874672085615616/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828484.616266:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828484.616269:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828484.616270:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828484.616273:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1919 00000100:00080000:1.0:1787828484.616278:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1787828484.616286:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787828484.616288:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828484.616294:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:1.0:1787828484.616305:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f43c0 x1874672085615744/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828484.616312:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828484.616316:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828484.616318:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828484.616321:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1919 00000100:00080000:1.0:1787828484.616324:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828484.616327:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828484.616332:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:1.0:1787828484.616357:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4000 x1874672085615872/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787828484.616359:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828484/real 1787828484] req@ffff9e35da6f5e00 x1874672085615360/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828509 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828484.616409:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828484.616418:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828484.616420:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828484.616426:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1919 00000100:00100000:0.0:1787828484.616430:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5e00 x1874672085615360/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828509 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828484.616432:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828484.616443:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828484.616456:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1787828484.616469:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1787828484.616476:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7480 x1874672085616000/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828484.616479:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828484.616493:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085615360:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787828484.616500:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828484.616504:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787828484.616506:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828484.616510:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1919 00000100:00080000:1.0:1787828484.616513:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828484.616516:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828484.616519:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828484.616531:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085615488:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828484.616533:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7840 x1874672085616128/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828484.616567:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828484.616571:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828484.616574:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828484.616592:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1919 00000100:00080000:1.0:1787828484.616596:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1787828484.616618:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787828484.616623:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787828484.616644:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828484/real 1787828484] req@ffff9e35da6f6d00 x1874672085615488/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828509 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1787828484.616647:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:0.0:1787828484.616683:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6d00 x1874672085615488/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828509 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828484.616697:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1787828484.616705:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4f00 x1874672085616256/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828484.616706:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828484.616717:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085615488:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787828484.616746:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085615616:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787828484.616753:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828484.616758:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828484.616762:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828484.616770:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1919 00000800:00000100:0.0:1787828484.616792:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787828484.616794:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828484.616801:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828484.616844:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:0.0:1787828484.616858:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828484/real 1787828484] req@ffff9e35da6f4b40 x1874672085615616/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828509 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828484.616859:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4780 x1874672085616384/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787828484.616947:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4b40 x1874672085615616/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828509 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828484.616958:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828484.616961:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828484.616964:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085615616:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787828484.616970:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085615744:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787828484.616989:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828484.616998:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828484/real 1787828484] req@ffff9e35da6f43c0 x1874672085615744/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828509 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828484.617008:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f43c0 x1874672085615744/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828509 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828484.617017:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828484.617021:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828484.617024:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085615744:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787828484.617031:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085615872:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787828484.617054:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828484.617083:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828484/real 1787828484] req@ffff9e35da6f4000 x1874672085615872/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828509 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828484.617093:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4000 x1874672085615872/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828509 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828484.617105:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828484.617108:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828484.617110:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085615872:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787828484.617115:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085616000:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787828484.617132:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828484.617140:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828484/real 1787828484] req@ffff9e35da6f7480 x1874672085616000/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828509 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828484.617149:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7480 x1874672085616000/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828509 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828484.617155:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828484.617157:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828484.617159:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085616000:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787828484.617178:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085616128:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787828484.617190:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828484.617197:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828484/real 1787828484] req@ffff9e35da6f7840 x1874672085616128/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828509 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828484.617206:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7840 x1874672085616128/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828509 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828484.617213:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828484.617217:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828484.617221:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085616128:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787828484.617228:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085616256:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787828484.617251:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828484.617265:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828484/real 1787828484] req@ffff9e35da6f4f00 x1874672085616256/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828509 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828484.617279:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4f00 x1874672085616256/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828509 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828484.617286:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828484.617288:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828484.617291:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085616256:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787828484.617295:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085616384:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787828484.617311:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828484.617319:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828484/real 1787828484] req@ffff9e35da6f4780 x1874672085616384/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828509 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828484.617327:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4780 x1874672085616384/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828509 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828484.617333:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828484.617335:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828484.617354:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085616384:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787828484.743937:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828484.743968:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828484.744037:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787828484.744105:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828485.767857:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828485.768012:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787828485.768017:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828485.768108:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828486.791882:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828486.791903:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828486.791995:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828486.791997:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828487.815960:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828487.815964:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787828487.815983:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787828487.816102:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828488.839927:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787828488.839938:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828488.839943:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828488.840049:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828489.736001:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828489.736022:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828489.736030:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828489.736051:0:27606:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1925 00000100:00080000:1.0:1787828489.736072:0:27606:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828489.736118:0:27606:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787828489.736202:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f61c0 x1874672085616512/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828489.736274:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828489.736290:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828489.736311:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828489.736326:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1925 00000100:00080000:1.0:1787828489.736332:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828489.736338:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828489.736353:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828489.736395:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085616512:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1787828489.736405:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5a40 x1874672085616640/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828489.736425:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828489.736431:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787828489.736434:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828489.736440:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1925 00000100:00080000:1.0:1787828489.736445:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828489.736450:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828489.736457:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828489.736475:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6580 x1874672085616768/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828489.736494:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1787828489.736506:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787828489.736507:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828489.736513:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828489.736542:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1925 00000100:00080000:1.0:1787828489.736551:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828489.736560:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828489.736570:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:1.0:1787828489.736592:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6940 x1874672085616896/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828489.736608:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828489.736613:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:0.0:1787828489.736618:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828489/real 1787828489] req@ffff9e35da6f61c0 x1874672085616512/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828514 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828489.736618:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828489.736632:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1925 00000100:00080000:1.0:1787828489.736641:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828489.736649:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828489.736656:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:0.0:1787828489.736664:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f61c0 x1874672085616512/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828514 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828489.736694:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f70c0 x1874672085617024/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828489.736734:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828489.736736:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828489.736748:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828489.736756:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828489.736766:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1925 00000100:00080000:1.0:1787828489.736797:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828489.736857:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828489.736870:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1787828489.736877:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828489.736899:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085616512:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1787828489.736900:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4780 x1874672085617152/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828489.736927:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828489.736932:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787828489.736935:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828489.736941:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1925 00000100:00080000:1.0:1787828489.736947:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828489.736952:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828489.736960:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828489.736962:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085616640:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828489.736993:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4f00 x1874672085617280/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828489.737019:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1787828489.737026:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787828489.737036:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828489.737040:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828489.737051:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1925 00000100:00080000:1.0:1787828489.737062:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787828489.737071:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828489/real 1787828489] req@ffff9e35da6f5a40 x1874672085616640/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828514 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828489.737079:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828489.737115:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:0.0:1787828489.737124:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5a40 x1874672085616640/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828514 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828489.737137:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1787828489.737138:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7840 x1874672085617408/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828489.737147:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828489.737159:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085616640:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787828489.737203:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787828489.737208:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085616768:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787828489.737222:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828489.737231:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828489.737243:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1925 00000800:00000100:0.0:1787828489.737259:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787828489.737271:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787828489.737285:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828489/real 1787828489] req@ffff9e35da6f6580 x1874672085616768/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828514 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828489.737288:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828489.737332:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:0.0:1787828489.737340:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6580 x1874672085616768/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828514 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828489.737352:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1787828489.737354:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7480 x1874672085617536/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828489.737379:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828489.737393:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085616768:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787828489.737411:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085616896:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787828489.737443:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828489.737459:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828489/real 1787828489] req@ffff9e35da6f6940 x1874672085616896/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828514 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828489.737474:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6940 x1874672085616896/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828514 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828489.737485:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828489.737489:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828489.737493:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085616896:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787828489.737500:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085617024:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787828489.737546:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828489.737556:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828489/real 1787828489] req@ffff9e35da6f70c0 x1874672085617024/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828514 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828489.737575:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f70c0 x1874672085617024/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828514 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828489.737594:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828489.737596:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828489.737599:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085617024:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787828489.737605:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085617152:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787828489.737621:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828489.737631:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828489/real 1787828489] req@ffff9e35da6f4780 x1874672085617152/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828514 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828489.737643:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4780 x1874672085617152/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828514 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828489.737680:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828489.737684:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828489.737687:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085617152:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787828489.737720:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085617280:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787828489.737744:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828489.737756:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828489/real 1787828489] req@ffff9e35da6f4f00 x1874672085617280/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828514 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828489.737869:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4f00 x1874672085617280/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828514 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828489.737901:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828489.737908:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828489.737915:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085617280:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787828489.737930:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085617408:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787828489.737990:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828489.738017:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828489/real 1787828489] req@ffff9e35da6f7840 x1874672085617408/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828514 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828489.738043:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7840 x1874672085617408/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828514 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828489.738061:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828489.738066:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828489.738070:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085617408:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787828489.738079:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085617536:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787828489.738122:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828489.738139:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828489/real 1787828489] req@ffff9e35da6f7480 x1874672085617536/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828514 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828489.738158:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7480 x1874672085617536/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828514 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828489.738190:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828489.738197:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828489.738206:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085617536:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828489.863919:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787828489.863965:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787828489.863965:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828489.864049:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828490.887894:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828490.887898:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787828490.887938:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828490.887965:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828491.911912:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828491.911927:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828491.911929:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787828491.911949:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828492.935874:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828492.935884:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787828492.935910:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787828492.935916:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828493.959964:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828493.959989:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787828493.960041:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:0.0:1787828493.960103:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828494.855943:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828494.855954:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828494.855959:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828494.855971:0:27606:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1930 00000100:00080000:1.0:1787828494.855987:0:27606:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828494.855999:0:27606:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787828494.856051:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4000 x1874672085617664/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828494.856097:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828494.856102:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828494.856104:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828494.856110:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1930 00000100:00080000:1.0:1787828494.856114:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828494.856118:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828494.856132:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828494.856154:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f43c0 x1874672085617792/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828494.856162:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085617664:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787828494.856177:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828494.856182:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787828494.856183:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828494.856190:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1930 00000100:00080000:1.0:1787828494.856194:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828494.856201:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828494.856207:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828494.856217:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4b40 x1874672085617920/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828494.856225:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828494.856227:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828494.856229:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828494.856233:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1930 00000100:00080000:1.0:1787828494.856236:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828494.856240:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828494.856243:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:1.0:1787828494.856254:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6d00 x1874672085618048/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828494.856262:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828494.856265:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828494.856266:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828494.856283:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1930 00000800:00000100:2.0:1787828494.856290:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787828494.856291:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828494.856301:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828494.856307:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:1.0:1787828494.856320:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5e00 x1874672085618176/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828494.856327:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828494.856330:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828494.856331:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828494.856336:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1930 00000100:00080000:1.0:1787828494.856341:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828494.856346:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828494.856355:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828494.856360:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828494/real 1787828494] req@ffff9e35da6f4000 x1874672085617664/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828519 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828494.856378:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7480 x1874672085618304/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828494.856389:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4000 x1874672085617664/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828519 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828494.856405:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828494.856430:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787828494.856439:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828494.856449:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1930 00000100:00080000:2.0:1787828494.856449:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828494.856465:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828494.856467:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:1.0:1787828494.856479:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787828494.856482:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085617664:192.168.203.141@tcp:250:kworker.0 00000008:00100000:1.0:1787828494.856491:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828494.856509:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7840 x1874672085618432/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828494.856524:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787828494.856525:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085617792:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787828494.856539:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828494.856543:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828494.856550:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1930 00000100:00080000:1.0:1787828494.856558:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828494.856563:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828494.856570:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000800:00000100:2.0:1787828494.856586:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828494.856588:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4f00 x1874672085618560/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828494.856603:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828494.856606:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828494.856608:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828494.856617:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1930 00000100:00100000:2.0:1787828494.856619:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828494/real 1787828494] req@ffff9e35da6f43c0 x1874672085617792/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828519 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828494.856622:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828494.856642:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828494.856659:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:2.0:1787828494.856677:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f43c0 x1874672085617792/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828519 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828494.856678:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4780 x1874672085618688/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828494.856761:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828494.856772:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828494.856780:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085617792:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828494.856792:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085617920:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828494.856876:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828494.856893:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828494/real 1787828494] req@ffff9e35da6f4b40 x1874672085617920/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828519 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828494.856908:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4b40 x1874672085617920/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828519 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828494.856918:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828494.856923:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828494.856928:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085617920:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828494.856938:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085618048:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828494.856961:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828494.856973:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828494/real 1787828494] req@ffff9e35da6f6d00 x1874672085618048/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828519 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828494.856987:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6d00 x1874672085618048/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828519 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828494.856996:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828494.856999:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828494.857002:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085618048:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828494.857008:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085618176:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828494.857024:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828494.857034:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828494/real 1787828494] req@ffff9e35da6f5e00 x1874672085618176/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828519 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828494.857046:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5e00 x1874672085618176/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828519 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828494.857055:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828494.857058:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828494.857061:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085618176:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828494.857067:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085618304:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828494.857083:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828494.857093:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828494/real 1787828494] req@ffff9e35da6f7480 x1874672085618304/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828519 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828494.857106:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7480 x1874672085618304/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828519 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828494.857114:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828494.857118:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828494.857121:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085618304:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828494.857182:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085618432:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828494.857234:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828494.857255:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828494/real 1787828494] req@ffff9e35da6f7840 x1874672085618432/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828519 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828494.857278:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7840 x1874672085618432/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828519 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828494.857312:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828494.857320:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828494.857326:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085618432:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828494.857339:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085618560:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828494.857372:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828494.857387:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828494/real 1787828494] req@ffff9e35da6f4f00 x1874672085618560/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828519 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828494.857402:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4f00 x1874672085618560/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828519 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828494.857421:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828494.857425:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828494.857429:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085618560:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828494.857437:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085618688:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828494.857455:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828494.857466:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828494/real 1787828494] req@ffff9e35da6f4780 x1874672085618688/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828519 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828494.857480:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4780 x1874672085618688/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828519 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828494.857507:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828494.857511:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828494.857514:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085618688:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828494.983936:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787828494.983956:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828494.983960:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828494.983965:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828496.007873:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787828496.007880:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787828496.007914:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787828496.007968:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828497.031908:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828497.031918:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828497.031991:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787828497.032019:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828498.055989:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787828498.055990:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828498.056032:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828498.056037:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828499.079911:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828499.079938:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787828499.079939:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828499.080032:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828499.975938:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828499.975961:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828499.975967:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828499.975979:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1935 00000100:00080000:0.0:1787828499.975992:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828499.976000:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787828499.976046:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718000 x1874672085618816/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828499.976081:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828499.976086:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828499.976089:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828499.976093:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1935 00000100:00080000:0.0:1787828499.976097:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828499.976101:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828499.976113:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828499.976130:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718780 x1874672085618944/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828499.976150:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787828499.976151:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085618816:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1787828499.976156:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787828499.976163:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828499.976172:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1935 00000100:00080000:0.0:1787828499.976176:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828499.976179:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828499.976185:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828499.976201:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da719a40 x1874672085619072/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828499.976212:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828499.976217:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787828499.976219:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828499.976223:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1935 00000100:00080000:0.0:1787828499.976228:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828499.976232:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828499.976237:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:0.0:1787828499.976251:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a580 x1874672085619200/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787828499.976262:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787828499.976275:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828499.976280:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787828499.976283:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828499.976290:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1935 00000100:00080000:0.0:1787828499.976296:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828499.976317:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828499.976324:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:0.0:1787828499.976343:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7183c0 x1874672085619328/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828499.976350:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828499/real 1787828499] req@ffff9e35da718000 x1874672085618816/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828524 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828499.976380:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828499.976388:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828499.976392:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828499.976402:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1935 00000100:00100000:2.0:1787828499.976403:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718000 x1874672085618816/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828524 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828499.976408:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828499.976420:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:2.0:1787828499.976438:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:0.0:1787828499.976445:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1787828499.976453:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828499.976470:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a1c0 x1874672085619456/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828499.976472:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085618816:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1787828499.976501:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828499.976505:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787828499.976508:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828499.976515:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1935 00000100:00080000:0.0:1787828499.976518:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828499.976523:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828499.976530:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828499.976534:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085618944:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787828499.976564:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a940 x1874672085619584/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828499.976588:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828499.976592:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787828499.976596:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828499.976601:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1935 00000100:00080000:0.0:1787828499.976610:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1787828499.976611:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787828499.976623:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828499.976632:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:2.0:1787828499.976652:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828499/real 1787828499] req@ffff9e35da718780 x1874672085618944/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828524 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828499.976653:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718f00 x1874672085619712/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828499.976729:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787828499.976738:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718780 x1874672085618944/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828524 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828499.976739:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787828499.976746:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828499.976762:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1935 00000100:00080000:2.0:1787828499.976768:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828499.976770:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828499.976781:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:0.0:1787828499.976783:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787828499.976793:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085618944:192.168.203.141@tcp:38:kworker.0 00000008:00100000:0.0:1787828499.976795:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:0.0:1787828499.976854:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718b40 x1874672085619840/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828499.976867:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085619072:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828499.976918:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828499.976936:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828499/real 1787828499] req@ffff9e35da719a40 x1874672085619072/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828524 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828499.976953:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672085619072/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828524 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828499.976971:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828499.976975:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828499.976979:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085619072:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828499.976986:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085619200:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828499.977008:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828499.977019:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828499/real 1787828499] req@ffff9e35da71a580 x1874672085619200/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828524 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828499.977036:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672085619200/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828524 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828499.977052:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828499.977058:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828499.977064:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085619200:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828499.977075:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085619328:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828499.977110:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828499.977131:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828499/real 1787828499] req@ffff9e35da7183c0 x1874672085619328/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828524 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828499.977154:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672085619328/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828524 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828499.977168:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828499.977174:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828499.977180:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085619328:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828499.977191:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085619456:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828499.977228:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828499.977250:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828499/real 1787828499] req@ffff9e35da71a1c0 x1874672085619456/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828524 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828499.977273:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a1c0 x1874672085619456/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828524 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828499.977285:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828499.977288:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828499.977292:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085619456:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828499.977321:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085619584:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828499.977348:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828499.977361:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828499/real 1787828499] req@ffff9e35da71a940 x1874672085619584/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828524 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828499.977374:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672085619584/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828524 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828499.977384:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828499.977387:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828499.977391:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085619584:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828499.977397:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085619712:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828499.977414:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828499.977427:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828499/real 1787828499] req@ffff9e35da718f00 x1874672085619712/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828524 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828499.977449:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672085619712/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828524 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828499.977464:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828499.977470:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828499.977476:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085619712:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828499.977487:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085619840:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828499.977523:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828499.977541:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828499/real 1787828499] req@ffff9e35da718b40 x1874672085619840/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828524 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828499.977557:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718b40 x1874672085619840/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828524 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828499.977573:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828499.977579:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828499.977584:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085619840:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828500.103903:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787828500.103904:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787828500.103941:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787828500.104043:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828501.127912:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787828501.127953:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787828501.128029:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828501.128112:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828502.152004:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787828502.152005:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828502.152075:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828502.152183:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828503.175929:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787828503.176022:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787828503.176022:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787828503.176189:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828504.199939:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787828504.199949:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787828504.200027:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828504.200215:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828505.095903:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828505.095930:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787828505.095934:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828505.095943:0:27606:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1940 00000100:00080000:3.0:1787828505.095954:0:27606:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828505.095972:0:27606:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787828505.096007:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499a40 x1874672085619968/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828505.096042:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828505.096046:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828505.096047:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828505.096051:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1940 00000100:00080000:3.0:1787828505.096054:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828505.096058:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828505.096076:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828505.096085:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085619968:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1787828505.096092:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649ad00 x1874672085620096/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828505.096118:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828505.096120:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787828505.096122:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828505.096127:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1940 00000100:00080000:3.0:1787828505.096130:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828505.096133:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828505.096138:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828505.096150:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b0c0 x1874672085620224/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828505.096160:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828505.096163:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787828505.096164:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828505.096167:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1940 00000100:00080000:3.0:1787828505.096171:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1787828505.096175:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1787828505.096179:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828505.096186:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:3.0:1787828505.096198:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b480 x1874672085620352/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828505.096206:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828505.096209:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787828505.096212:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828505.096217:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1940 00000100:00100000:2.0:1787828505.096222:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828505/real 1787828505] req@ffff9e35c6499a40 x1874672085619968/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828530 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828505.096223:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828505.096232:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828505.096241:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:3.0:1787828505.096255:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499e00 x1874672085620480/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828505.096260:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499a40 x1874672085619968/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828530 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828505.096276:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828505.096282:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828505.096283:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828505.096284:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828505.096295:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:3.0:1787828505.096296:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1940 00000100:00080000:3.0:1787828505.096305:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787828505.096309:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085619968:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1787828505.096312:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828505.096323:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828505.096334:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c64983c0 x1874672085620608/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828505.096344:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787828505.096345:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085620096:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1787828505.096348:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787828505.096353:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828505.096358:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1940 00000100:00080000:3.0:1787828505.096361:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828505.096365:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828505.096370:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1787828505.096378:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828505.096384:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a1c0 x1874672085620736/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828505.096394:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787828505.096395:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828505/real 1787828505] req@ffff9e35c649ad00 x1874672085620096/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828530 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828505.096400:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787828505.096405:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828505.096416:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1940 00000100:00080000:3.0:1787828505.096423:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787828505.096427:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649ad00 x1874672085620096/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828530 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828505.096431:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828505.096438:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00080000:2.0:1787828505.096448:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1787828505.096455:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498000 x1874672085620864/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828505.096458:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828505.096468:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085620096:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1787828505.096478:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787828505.096484:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085620224:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1787828505.096485:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787828505.096490:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828505.096504:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1940 00000100:00080000:3.0:1787828505.096508:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828505.096514:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:2.0:1787828505.096520:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:3.0:1787828505.096525:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:2.0:1787828505.096533:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828505/real 1787828505] req@ffff9e35c649b0c0 x1874672085620224/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828530 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828505.096563:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c64992c0 x1874672085620992/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828505.096580:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b0c0 x1874672085620224/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828530 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828505.096592:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828505.096595:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828505.096597:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085620224:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828505.096601:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085620352:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828505.096616:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828505.096622:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828505/real 1787828505] req@ffff9e35c649b480 x1874672085620352/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828530 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828505.096630:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b480 x1874672085620352/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828530 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828505.096635:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828505.096637:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828505.096639:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085620352:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828505.096642:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085620480:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828505.096652:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828505.096658:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828505/real 1787828505] req@ffff9e35c6499e00 x1874672085620480/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828530 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828505.096679:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499e00 x1874672085620480/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828530 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828505.096700:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828505.096702:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828505.096705:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085620480:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828505.096719:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c64983c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085620608:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828505.096732:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828505.096738:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828505/real 1787828505] req@ffff9e35c64983c0 x1874672085620608/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828530 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828505.096746:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c64983c0 x1874672085620608/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828530 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828505.096762:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828505.096765:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828505.096767:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c64983c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085620608:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828505.096770:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085620736:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828505.096779:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828505.096785:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828505/real 1787828505] req@ffff9e35c649a1c0 x1874672085620736/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828530 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828505.096792:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a1c0 x1874672085620736/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828530 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828505.096826:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828505.096828:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828505.096831:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085620736:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828505.096846:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085620864:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828505.096859:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828505.096866:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828505/real 1787828505] req@ffff9e35c6498000 x1874672085620864/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828530 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828505.096874:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498000 x1874672085620864/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828530 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828505.096880:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828505.096882:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828505.096884:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085620864:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828505.096887:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c64992c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085620992:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828505.096917:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828505.096930:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828505/real 1787828505] req@ffff9e35c64992c0 x1874672085620992/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828530 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828505.096946:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c64992c0 x1874672085620992/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828530 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828505.096958:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828505.096962:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828505.096967:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c64992c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085620992:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787828505.223916:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787828505.223916:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787828505.223918:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828505.224001:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828506.247953:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787828506.247997:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828506.248034:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787828506.248123:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828507.271932:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828507.271971:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787828507.272004:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828507.272013:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828508.295967:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787828508.295989:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828508.295997:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828508.296099:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828509.319952:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828509.319962:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787828509.319970:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787828509.320063:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828510.215997:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828510.216011:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828510.216017:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828510.216033:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1945 00000100:00080000:1.0:1787828510.216053:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828510.216066:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787828510.216133:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f70c0 x1874672085621120/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828510.216185:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828510.216191:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828510.216194:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828510.216199:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1945 00000100:00080000:1.0:1787828510.216205:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828510.216210:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828510.216225:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828510.216246:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6940 x1874672085621248/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828510.216260:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828510.216264:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787828510.216267:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828510.216272:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1945 00000100:00080000:1.0:1787828510.216275:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828510.216279:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828510.216284:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828510.216298:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6580 x1874672085621376/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828510.216316:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085621120:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787828510.216320:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828510.216340:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828510.216344:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828510.216350:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1945 00000100:00080000:1.0:1787828510.216354:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828510.216360:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828510.216366:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:1.0:1787828510.216384:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5a40 x1874672085621504/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828510.216396:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828510.216401:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828510.216405:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828510.216412:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1945 00000100:00080000:1.0:1787828510.216417:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1787828510.216419:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787828510.216430:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828510.216438:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:1.0:1787828510.216454:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f61c0 x1874672085621632/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828510.216467:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828510.216472:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828510.216475:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787828510.216483:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828510/real 1787828510] req@ffff9e35da6f70c0 x1874672085621120/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828535 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828510.216488:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1945 00000100:00080000:1.0:1787828510.216497:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828510.216513:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828510.216524:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828510.216531:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f70c0 x1874672085621120/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828535 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828510.216542:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4780 x1874672085621760/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828510.216554:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828510.216568:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828510.216570:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:1.0:1787828510.216583:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:2.0:1787828510.216586:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085621120:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787828510.216588:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828510.216606:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1945 00000100:00100000:2.0:1787828510.216612:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085621248:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787828510.216616:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828510.216624:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828510.216663:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1787828510.216684:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828510.216701:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4f00 x1874672085621888/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828510.216709:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828510/real 1787828510] req@ffff9e35da6f6940 x1874672085621248/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828535 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828510.216764:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828510.216770:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:2.0:1787828510.216773:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6940 x1874672085621248/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828535 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828510.216775:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828510.216809:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828510.216815:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828510.216819:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085621248:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787828510.216842:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1945 00000100:00080000:1.0:1787828510.216863:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828510.216870:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828510.216877:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:1.0:1787828510.216895:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7840 x1874672085622016/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828510.216907:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828510.216911:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828510.216914:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828510.216919:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1945 00000100:00080000:1.0:1787828510.216935:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828510.216941:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828510.216947:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:2.0:1787828510.216953:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085621376:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828510.216966:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7480 x1874672085622144/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787828510.217008:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828510.217033:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828510/real 1787828510] req@ffff9e35da6f6580 x1874672085621376/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828535 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828510.217051:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6580 x1874672085621376/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828535 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828510.217062:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828510.217066:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828510.217070:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085621376:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828510.217078:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085621504:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828510.217098:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828510.217110:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828510/real 1787828510] req@ffff9e35da6f5a40 x1874672085621504/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828535 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828510.217124:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5a40 x1874672085621504/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828535 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828510.217134:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828510.217138:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828510.217141:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085621504:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828510.217148:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085621632:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828510.217167:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828510.217178:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828510/real 1787828510] req@ffff9e35da6f61c0 x1874672085621632/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828535 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828510.217192:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f61c0 x1874672085621632/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828535 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828510.217211:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828510.217231:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828510.217234:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085621632:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828510.217239:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085621760:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828510.217255:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828510.217263:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828510/real 1787828510] req@ffff9e35da6f4780 x1874672085621760/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828535 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828510.217272:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4780 x1874672085621760/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828535 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828510.217279:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828510.217281:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828510.217283:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085621760:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828510.217288:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085621888:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828510.217301:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828510.217309:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828510/real 1787828510] req@ffff9e35da6f4f00 x1874672085621888/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828535 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828510.217318:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4f00 x1874672085621888/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828535 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828510.217323:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828510.217326:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828510.217329:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085621888:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828510.217333:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085622016:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828510.217346:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828510.217353:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828510/real 1787828510] req@ffff9e35da6f7840 x1874672085622016/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828535 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828510.217361:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7840 x1874672085622016/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828535 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828510.217368:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828510.217370:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828510.217372:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085622016:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828510.217394:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085622144:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828510.217407:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828510.217415:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828510/real 1787828510] req@ffff9e35da6f7480 x1874672085622144/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828535 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828510.217424:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7480 x1874672085622144/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828535 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828510.217431:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828510.217433:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828510.217435:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085622144:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787828510.344119:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828510.344153:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828510.344217:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828510.344297:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828511.367899:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828511.367983:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828511.368031:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828511.368175:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787828512.392013:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828512.392014:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828512.392017:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787828512.392172:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828513.415961:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828513.416021:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:3.0:1787828513.416021:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828513.416157:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828514.439890:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828514.439928:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828514.439995:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828514.440008:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00080000:2.0:1787828515.335930:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828515.335946:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828515.335952:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828515.335966:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1950 00000100:00080000:2.0:1787828515.335981:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828515.335992:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787828515.336037:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f840 x1874672085622272/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828515.336075:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828515.336080:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828515.336082:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828515.336086:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1950 00000100:00080000:2.0:1787828515.336091:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828515.336095:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828515.336108:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828515.336142:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f0c0 x1874672085622400/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787828515.336164:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085622272:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1787828515.336197:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828515.336204:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787828515.336208:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828515.336218:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1950 00000100:00080000:2.0:1787828515.336227:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828515.336232:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828515.336239:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828515.336262:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f480 x1874672085622528/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787828515.336267:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1787828515.336284:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828515.336289:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787828515.336292:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828515.336298:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1950 00000100:00080000:2.0:1787828515.336303:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828515.336308:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828515.336313:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:2.0:1787828515.336328:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e1c0 x1874672085622656/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828515.336340:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828515.336349:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:0.0:1787828515.336351:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828515/real 1787828515] req@ffff9e35c498f840 x1874672085622272/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828540 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828515.336352:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828515.336363:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1950 00000100:00080000:2.0:1787828515.336371:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828515.336382:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828515.336394:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:0.0:1787828515.336434:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f840 x1874672085622272/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828540 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828515.336453:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cb40 x1874672085622784/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828515.336479:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828515.336491:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828515.336498:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:2.0:1787828515.336499:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828515.336506:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787828515.336515:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085622272:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1787828515.336516:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1950 00000100:00080000:2.0:1787828515.336524:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828515.336552:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828515.336559:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828515.336573:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c000 x1874672085622912/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787828515.336583:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085622400:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1787828515.336617:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828515.336634:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787828515.336637:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828515.336662:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1950 00000100:00080000:2.0:1787828515.336673:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1787828515.336684:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1787828515.336687:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828515.336712:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828515.336725:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828515/real 1787828515] req@ffff9e35c498f0c0 x1874672085622400/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828540 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828515.336745:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f840 x1874672085623040/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787828515.336751:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f0c0 x1874672085622400/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828540 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828515.336798:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828515.336802:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828515.336806:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085622400:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787828515.336817:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085622528:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1787828515.336862:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828515.336867:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787828515.336870:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828515.336877:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1950 00000100:00080000:2.0:1787828515.336886:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828515.336896:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:0.0:1787828515.336898:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:2.0:1787828515.336911:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:0.0:1787828515.336922:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828515/real 1787828515] req@ffff9e35c498f480 x1874672085622528/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828540 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828515.336940:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e580 x1874672085623168/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787828515.336959:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f480 x1874672085622528/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828540 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828515.336971:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828515.336975:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828515.336982:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787828515.336985:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:2.0:1787828515.336990:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787828515.336994:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085622528:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1787828515.337004:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1950 00000100:00100000:0.0:1787828515.337010:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085622656:192.168.203.141@tcp:8:kworker.0 00000100:00080000:2.0:1787828515.337014:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828515.337024:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828515.337032:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:2.0:1787828515.337052:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cf00 x1874672085623296/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787828515.337066:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828515.337099:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828515/real 1787828515] req@ffff9e35c498e1c0 x1874672085622656/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828540 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828515.337123:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e1c0 x1874672085622656/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828540 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828515.337134:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828515.337137:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828515.337141:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085622656:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787828515.337178:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085622784:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787828515.337204:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828515.337218:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828515/real 1787828515] req@ffff9e35c498cb40 x1874672085622784/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828540 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828515.337239:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498cb40 x1874672085622784/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828540 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828515.337254:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828515.337260:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828515.337266:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085622784:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787828515.337302:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085622912:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787828515.337337:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828515.337358:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828515/real 1787828515] req@ffff9e35c498c000 x1874672085622912/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828540 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828515.337381:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c000 x1874672085622912/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828540 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828515.337397:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828515.337403:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828515.337426:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085622912:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787828515.337448:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085623040:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787828515.337483:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828515.337502:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828515/real 1787828515] req@ffff9e35c498f840 x1874672085623040/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828540 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828515.337524:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f840 x1874672085623040/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828540 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828515.337557:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828515.337561:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828515.337565:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085623040:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787828515.337572:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085623168:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787828515.337627:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828515.337663:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828515/real 1787828515] req@ffff9e35c498e580 x1874672085623168/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828540 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828515.337678:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e580 x1874672085623168/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828540 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828515.337699:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828515.337703:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828515.337707:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085623168:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787828515.337715:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085623296:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787828515.337737:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828515.337748:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828515/real 1787828515] req@ffff9e35c498cf00 x1874672085623296/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828540 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828515.337780:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498cf00 x1874672085623296/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828540 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828515.337789:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828515.337792:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828515.337796:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085623296:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787828515.463874:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828515.463874:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787828515.463882:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787828515.463957:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828516.487898:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787828516.487899:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828516.487912:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828516.487984:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828517.511925:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828517.512025:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828517.512211:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:3.0:1787828517.512237:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828518.535914:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787828518.535958:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:3.0:1787828518.535958:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:1.0:1787828518.536126:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828519.559883:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828519.559883:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787828519.559944:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828519.559946:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00080000:1.0:1787828520.455893:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828520.455905:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828520.455908:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828520.455917:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1955 00000100:00080000:1.0:1787828520.455927:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828520.455933:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787828520.455965:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5e00 x1874672085623424/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828520.455989:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828520.455992:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828520.455994:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828520.455997:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1955 00000100:00080000:1.0:1787828520.456000:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828520.456002:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828520.456010:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828520.456019:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6d00 x1874672085623552/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828520.456027:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828520.456029:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787828520.456031:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828520.456033:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1955 00000100:00080000:1.0:1787828520.456035:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828520.456039:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828520.456043:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828520.456050:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4b40 x1874672085623680/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828520.456059:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828520.456062:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828520.456063:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787828520.456069:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085623424:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787828520.456069:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1955 00000100:00080000:1.0:1787828520.456071:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828520.456079:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828520.456083:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:1.0:1787828520.456092:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f43c0 x1874672085623808/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828520.456099:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828520.456103:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828520.456104:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828520.456108:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1955 00000100:00080000:1.0:1787828520.456124:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828520.456127:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828520.456131:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:1.0:1787828520.456140:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4000 x1874672085623936/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828520.456147:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828520.456150:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828520.456152:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828520.456156:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1955 00000100:00080000:1.0:1787828520.456160:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828520.456163:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828520.456167:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828520.456168:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085623552:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828520.456179:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7480 x1874672085624064/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828520.456188:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828520.456191:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787828520.456193:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828520.456197:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1955 00000100:00080000:1.0:1787828520.456201:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828520.456203:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828520.456207:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828520.456211:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085623680:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828520.456220:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7840 x1874672085624192/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828520.456230:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787828520.456235:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085623808:192.168.203.141@tcp:8:kworker.0 00000100:00080000:1.0:1787828520.456236:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828520.456240:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828520.456247:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1955 00000100:00080000:1.0:1787828520.456251:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828520.456254:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828520.456259:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:0.0:1787828520.456266:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085623936:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828520.456272:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4f00 x1874672085624320/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828520.456281:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828520.456284:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828520.456286:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787828520.456290:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085624064:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787828520.456300:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1955 00000100:00080000:1.0:1787828520.456306:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828520.456310:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787828520.456315:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085624192:192.168.203.141@tcp:38:kworker.0 00000008:00100000:1.0:1787828520.456316:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:1.0:1787828520.456334:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4780 x1874672085624448/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787828520.456343:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085624320:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787828520.456361:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085624448:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828520.583884:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787828520.583913:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828520.583946:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:0.0:1787828520.583969:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828521.607884:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787828521.607890:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828521.607908:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828521.607922:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828522.631922:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787828522.631972:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828522.632037:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828522.632039:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000400:00000100:0.0:1787828523.528324:0:2397:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.141:988 00000400:00000100:0.0:1787828523.528408:0:2397:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.141@tcp at host 192.168.203.141 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1787828523.528454:0:2397:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:0.0:1787828523.528575:0:2397:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000800:00000100:0.0:1787828523.528614:0:2397:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:1.0:1787828523.528628:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828520/real 1787828523] req@ffff9e35da6f4780 x1874672085624448/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828545 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:0.0:1787828523.528641:0:2397:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:0.0:1787828523.528667:0:2397:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:0.0:1787828523.528687:0:2397:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:1.0:1787828523.528720:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4780 x1874672085624448/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828545 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:0.0:1787828523.528740:0:2397:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00080000:1.0:1787828523.528763:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000800:00000100:0.0:1787828523.528777:0:2397:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00080000:1.0:1787828523.528786:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828523.528882:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085624448:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828523.528901:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828520/real 1787828523] req@ffff9e35da6f7840 x1874672085624192/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828545 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:0.0:1787828523.528914:0:2397:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:1.0:1787828523.528962:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7840 x1874672085624192/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828545 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828523.528980:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828523.528988:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828523.528996:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085624192:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828523.529009:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828520/real 1787828523] req@ffff9e35da6f4f00 x1874672085624320/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828545 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828523.529035:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4f00 x1874672085624320/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828545 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828523.529053:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828523.529059:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828523.529066:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085624320:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828523.529079:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828520/real 1787828523] req@ffff9e35da6f4b40 x1874672085623680/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828545 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828523.529141:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4b40 x1874672085623680/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828545 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828523.529157:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828523.529163:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828523.529168:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085623680:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828523.529180:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828520/real 1787828523] req@ffff9e35da6f43c0 x1874672085623808/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828545 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828523.529203:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f43c0 x1874672085623808/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828545 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828523.529216:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828523.529221:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828523.529226:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085623808:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828523.529237:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828520/real 1787828523] req@ffff9e35da6f4000 x1874672085623936/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828545 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828523.529259:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4000 x1874672085623936/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828545 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828523.529271:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828523.529277:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828523.529282:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085623936:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828523.529294:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828520/real 1787828523] req@ffff9e35da6f7480 x1874672085624064/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828545 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828523.529314:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7480 x1874672085624064/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828545 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828523.529327:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828523.529333:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828523.529338:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085624064:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828523.529349:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828520/real 1787828523] req@ffff9e35da6f5e00 x1874672085623424/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828545 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828523.529369:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5e00 x1874672085623424/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828545 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828523.529382:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828523.529388:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828523.529394:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085623424:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1787828523.529405:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828520/real 1787828523] req@ffff9e35da6f6d00 x1874672085623552/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828545 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828523.529428:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6d00 x1874672085623552/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828545 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828523.529444:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828523.529450:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828523.529455:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085623552:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828523.655925:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787828523.655928:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828523.655928:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:0.0:1787828523.656017:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828524.679887:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787828524.679895:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828524.679933:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828524.679985:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828525.575965:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828525.575981:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828525.575988:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828525.576003:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1960 00000100:00080000:1.0:1787828525.576017:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828525.576027:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787828525.576074:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6d00 x1874672085624576/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828525.576123:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828525.576127:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828525.576129:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828525.576134:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1960 00000100:00080000:1.0:1787828525.576138:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828525.576143:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828525.576156:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828525.576173:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5e00 x1874672085624704/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787828525.576206:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085624576:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787828525.576209:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828525.576221:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787828525.576228:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828525.576235:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1960 00000100:00080000:1.0:1787828525.576239:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828525.576243:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828525.576250:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828525.576264:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7480 x1874672085624832/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828525.576276:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828525.576281:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828525.576283:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828525.576288:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1960 00000100:00080000:1.0:1787828525.576322:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828525.576327:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:0.0:1787828525.576337:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:1.0:1787828525.576344:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:1.0:1787828525.576366:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4000 x1874672085624960/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828525.576412:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828525.576419:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828525.576423:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787828525.576428:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828525/real 1787828525] req@ffff9e35da6f6d00 x1874672085624576/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828550 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828525.576437:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1960 00000100:00080000:1.0:1787828525.576450:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828525.576477:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828525.576492:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:0.0:1787828525.576503:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6d00 x1874672085624576/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828550 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828525.576563:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1787828525.576566:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f43c0 x1874672085625088/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828525.576580:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828525.576610:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085624576:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787828525.576612:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828525.576627:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828525.576630:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828525.576637:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1960 00000100:00080000:1.0:1787828525.576643:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828525.576648:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828525.576654:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828525.576668:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4b40 x1874672085625216/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787828525.576676:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085624704:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787828525.576705:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828525.576710:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000800:00000100:0.0:1787828525.576716:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787828525.576716:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828525.576727:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1960 00000100:00080000:1.0:1787828525.576734:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787828525.576738:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828525/real 1787828525] req@ffff9e35da6f5e00 x1874672085624704/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828550 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828525.576745:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828525.576758:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828525.576768:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5e00 x1874672085624704/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828550 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828525.576783:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4f00 x1874672085625344/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828525.576786:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828525.576839:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:1.0:1787828525.576846:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787828525.576856:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085624704:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787828525.576857:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828525.576863:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828525.576874:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1960 00000100:00100000:0.0:1787828525.576877:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085624832:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787828525.576881:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828525.576893:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828525.576901:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000800:00000100:0.0:1787828525.576922:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828525.576923:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7840 x1874672085625472/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828525.576940:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828525.576946:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:0.0:1787828525.576948:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828525/real 1787828525] req@ffff9e35da6f7480 x1874672085624832/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828550 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828525.576950:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828525.576963:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1960 00000100:00080000:1.0:1787828525.576974:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787828525.576984:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7480 x1874672085624832/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828550 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828525.576985:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828525.576996:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00080000:0.0:1787828525.577006:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828525.577015:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828525.577020:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4780 x1874672085625600/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787828525.577026:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085624832:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787828525.577048:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085624960:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787828525.577114:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828525.577138:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828525/real 1787828525] req@ffff9e35da6f4000 x1874672085624960/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828550 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828525.577160:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4000 x1874672085624960/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828550 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828525.577174:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828525.577179:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828525.577186:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085624960:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787828525.577213:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085625088:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787828525.577241:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828525.577256:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828525/real 1787828525] req@ffff9e35da6f43c0 x1874672085625088/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828550 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828525.577270:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f43c0 x1874672085625088/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828550 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828525.577282:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828525.577286:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828525.577291:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085625088:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787828525.577319:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085625216:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787828525.577348:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828525.577365:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828525/real 1787828525] req@ffff9e35da6f4b40 x1874672085625216/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828550 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828525.577381:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4b40 x1874672085625216/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828550 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828525.577392:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828525.577396:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828525.577401:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085625216:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787828525.577410:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085625344:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787828525.577435:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828525.577448:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828525/real 1787828525] req@ffff9e35da6f4f00 x1874672085625344/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828550 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828525.577458:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4f00 x1874672085625344/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828550 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828525.577466:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828525.577468:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828525.577471:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085625344:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787828525.577476:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085625472:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787828525.577490:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828525.577503:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828525/real 1787828525] req@ffff9e35da6f7840 x1874672085625472/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828550 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828525.577534:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7840 x1874672085625472/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828550 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828525.577547:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828525.577552:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828525.577557:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085625472:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787828525.577565:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085625600:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787828525.577595:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828525.577614:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828525/real 1787828525] req@ffff9e35da6f4780 x1874672085625600/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828550 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828525.577640:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4780 x1874672085625600/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828550 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828525.577654:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828525.577659:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828525.577664:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085625600:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787828525.703947:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828525.703947:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787828525.703955:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828525.704060:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828526.727882:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787828526.727885:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828526.727894:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787828526.727941:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828527.751979:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828527.752000:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828527.752017:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828527.752044:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828528.776006:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787828528.776011:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828528.776064:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828528.776069:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828529.799930:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828529.800010:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828529.800024:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:0.0:1787828529.800076:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828530.696065:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828530.696078:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828530.696095:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828530.696125:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1966 00000100:00080000:1.0:1787828530.696142:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828530.696152:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787828530.696211:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f61c0 x1874672085625728/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828530.696265:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828530.696268:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828530.696270:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828530.696274:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1966 00000100:00080000:1.0:1787828530.696277:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828530.696280:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828530.696291:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828530.696330:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5a40 x1874672085625856/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828530.696342:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828530.696346:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787828530.696348:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828530.696352:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1966 00000100:00080000:1.0:1787828530.696355:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828530.696377:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828530.696385:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828530.696388:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085625728:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1787828530.696403:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6580 x1874672085625984/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828530.696416:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828530.696421:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828530.696423:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828530.696427:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1966 00000100:00080000:1.0:1787828530.696431:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828530.696435:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828530.696441:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:1.0:1787828530.696454:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6940 x1874672085626112/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828530.696465:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828530.696470:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828530.696472:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828530.696476:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1966 00000100:00080000:1.0:1787828530.696481:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1787828530.696515:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787828530.696516:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828530.696537:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:1.0:1787828530.696552:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f70c0 x1874672085626240/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828530.696565:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828530.696569:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828530.696572:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828530.696577:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1966 00000100:00080000:1.0:1787828530.696582:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828530.696589:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828530.696597:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828530.696611:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828530/real 1787828530] req@ffff9e35da6f61c0 x1874672085625728/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828555 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828530.696639:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f61c0 x1874672085625728/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828555 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828530.696646:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4780 x1874672085626368/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828530.696666:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828530.696679:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:1.0:1787828530.696684:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787828530.696689:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085625728:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787828530.696692:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787828530.696699:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787828530.696709:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085625856:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787828530.696710:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1966 00000100:00080000:1.0:1787828530.696748:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828530.696754:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828530.696761:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1787828530.696775:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828530.696781:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7840 x1874672085626496/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787828530.696811:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828530/real 1787828530] req@ffff9e35da6f5a40 x1874672085625856/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828555 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828530.696821:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828530.696887:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828530.696892:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828530.696900:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1966 00000100:00100000:0.0:1787828530.696905:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5a40 x1874672085625856/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828555 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828530.696908:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828530.696924:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:0.0:1787828530.696937:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828530.696945:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000008:00100000:1.0:1787828530.696947:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:0.0:1787828530.696951:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085625856:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828530.696978:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4f00 x1874672085626624/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787828530.697011:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085625984:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787828530.697011:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828530.697041:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828530.697043:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828530.697049:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1966 00000100:00080000:1.0:1787828530.697066:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828530.697070:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828530.697076:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000800:00000100:0.0:1787828530.697096:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828530.697097:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4b40 x1874672085626752/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787828530.697144:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828530/real 1787828530] req@ffff9e35da6f6580 x1874672085625984/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828555 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828530.697162:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6580 x1874672085625984/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828555 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828530.697173:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828530.697178:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828530.697183:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085625984:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787828530.697191:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085626112:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787828530.697235:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828530.697247:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828530/real 1787828530] req@ffff9e35da6f6940 x1874672085626112/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828555 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828530.697257:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6940 x1874672085626112/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828555 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828530.697272:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828530.697276:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828530.697280:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085626112:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787828530.697287:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085626240:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787828530.697346:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828530.697367:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828530/real 1787828530] req@ffff9e35da6f70c0 x1874672085626240/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828555 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828530.697390:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f70c0 x1874672085626240/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828555 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828530.697406:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828530.697410:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828530.697433:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085626240:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787828530.697443:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085626368:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787828530.697475:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828530.697519:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828530/real 1787828530] req@ffff9e35da6f4780 x1874672085626368/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828555 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828530.697549:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4780 x1874672085626368/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828555 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828530.697565:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828530.697571:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828530.697575:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085626368:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787828530.697583:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085626496:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787828530.697612:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828530.697625:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828530/real 1787828530] req@ffff9e35da6f7840 x1874672085626496/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828555 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828530.697637:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7840 x1874672085626496/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828555 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828530.697645:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828530.697648:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828530.697662:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085626496:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787828530.697696:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085626624:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787828530.697713:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828530.697735:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828530/real 1787828530] req@ffff9e35da6f4f00 x1874672085626624/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828555 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828530.697751:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4f00 x1874672085626624/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828555 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828530.697761:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828530.697764:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828530.697769:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085626624:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787828530.697793:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085626752:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787828530.697809:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828530.697859:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828530/real 1787828530] req@ffff9e35da6f4b40 x1874672085626752/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828555 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828530.697890:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4b40 x1874672085626752/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828555 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828530.697900:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828530.697905:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828530.697909:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085626752:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787828530.823953:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828530.823953:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828530.823992:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828530.824009:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828531.847951:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828531.848024:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787828531.848038:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787828531.848056:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828532.871941:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828532.871943:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828532.871976:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:0.0:1787828532.871987:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828533.895979:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828533.895980:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787828533.896041:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828533.896047:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828534.919927:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828534.919927:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828534.919995:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828534.920011:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00080000:0.0:1787828535.815939:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828535.815946:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828535.815950:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828535.815964:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1971 00000100:00080000:0.0:1787828535.815974:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828535.815980:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787828535.816012:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7192c0 x1874672085626880/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828535.816034:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828535.816037:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828535.816039:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828535.816042:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1971 00000100:00080000:0.0:1787828535.816045:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828535.816049:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828535.816057:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828535.816072:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718b40 x1874672085627008/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828535.816072:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085626880:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1787828535.816087:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828535.816089:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787828535.816090:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828535.816093:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1971 00000100:00080000:0.0:1787828535.816095:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828535.816098:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828535.816101:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828535.816111:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718f00 x1874672085627136/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828535.816117:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828535.816119:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787828535.816120:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828535.816123:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1971 00000100:00080000:0.0:1787828535.816126:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828535.816128:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828535.816132:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000800:00000100:1.0:1787828535.816135:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828535.816144:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a940 x1874672085627264/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828535.816152:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828535.816155:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787828535.816156:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828535.816159:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1971 00000100:00080000:0.0:1787828535.816161:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828535.816164:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828535.816167:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:1.0:1787828535.816173:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828535/real 1787828535] req@ffff9e35da7192c0 x1874672085626880/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828560 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828535.816177:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a1c0 x1874672085627392/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828535.816197:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787828535.816198:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7192c0 x1874672085626880/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828560 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828535.816204:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828535.816208:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828535.816216:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1971 00000100:00080000:1.0:1787828535.816217:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828535.816219:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828535.816223:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:0.0:1787828535.816227:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787828535.816232:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085626880:192.168.203.141@tcp:250:kworker.0 00000008:00100000:0.0:1787828535.816233:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828535.816248:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7183c0 x1874672085627520/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828535.816256:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787828535.816257:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085627008:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787828535.816262:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787828535.816265:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828535.816269:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1971 00000100:00080000:0.0:1787828535.816274:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828535.816277:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828535.816281:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1787828535.816282:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828535.816295:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a580 x1874672085627648/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828535.816297:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828535/real 1787828535] req@ffff9e35da718b40 x1874672085627008/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828560 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828535.816319:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828535.816324:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:1.0:1787828535.816324:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718b40 x1874672085627008/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828560 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828535.816328:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828535.816336:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1971 00000100:00080000:1.0:1787828535.816339:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828535.816341:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828535.816347:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:1.0:1787828535.816348:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000008:00100000:0.0:1787828535.816353:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:1.0:1787828535.816354:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085627008:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828535.816363:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085627136:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787828535.816366:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da719a40 x1874672085627776/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828535.816375:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828535.816377:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787828535.816379:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1787828535.816383:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787828535.816387:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1971 00000100:00080000:0.0:1787828535.816391:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828535.816394:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787828535.816398:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828535/real 1787828535] req@ffff9e35da718f00 x1874672085627136/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828560 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1787828535.816399:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:0.0:1787828535.816412:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718780 x1874672085627904/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828535.816414:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672085627136/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828560 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828535.816439:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828535.816459:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828535.816465:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085627136:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828535.816471:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085627264:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828535.816504:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828535.816513:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828535/real 1787828535] req@ffff9e35da71a940 x1874672085627264/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828560 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828535.816521:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672085627264/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828560 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828535.816528:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828535.816530:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828535.816533:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085627264:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828535.816537:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085627392:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828535.816552:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828535.816564:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828535/real 1787828535] req@ffff9e35da71a1c0 x1874672085627392/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828560 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828535.816595:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a1c0 x1874672085627392/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828560 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828535.816611:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828535.816615:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828535.816618:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085627392:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828535.816633:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085627520:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787828535.816648:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828535.816656:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828535/real 1787828535] req@ffff9e35da7183c0 x1874672085627520/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828560 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828535.816664:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672085627520/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828560 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828535.816670:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828535.816672:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828535.816674:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085627520:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828535.816677:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085627648:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787828535.816686:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828535.816692:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828535/real 1787828535] req@ffff9e35da71a580 x1874672085627648/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828560 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828535.816699:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672085627648/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828560 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828535.816704:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828535.816706:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828535.816709:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085627648:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828535.816713:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085627776:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828535.816722:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828535.816727:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828535/real 1787828535] req@ffff9e35da719a40 x1874672085627776/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828560 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828535.816734:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672085627776/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828560 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828535.816740:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828535.816741:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828535.816743:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085627776:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828535.816746:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085627904:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828535.816756:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828535.816761:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828535/real 1787828535] req@ffff9e35da718780 x1874672085627904/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828560 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828535.816769:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718780 x1874672085627904/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828560 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828535.816776:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828535.816778:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828535.816781:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085627904:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828535.943943:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787828535.943969:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787828535.943996:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787828535.944012:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828536.967975:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787828536.968002:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828536.968035:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828536.968112:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828537.991955:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828537.991983:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828537.991992:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828537.992069:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828539.015945:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828539.015970:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787828539.016095:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828539.016228:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828540.039889:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828540.039976:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787828540.040021:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828540.040021:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828540.935974:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828540.935985:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787828540.935989:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828540.935999:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1976 00000100:00080000:3.0:1787828540.936010:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828540.936017:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787828540.936057:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c64992c0 x1874672085628032/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828540.936086:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828540.936089:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828540.936091:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828540.936095:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1976 00000100:00080000:3.0:1787828540.936098:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828540.936102:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828540.936112:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828540.936125:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498000 x1874672085628160/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828540.936136:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828540.936140:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787828540.936142:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828540.936145:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1976 00000100:00080000:3.0:1787828540.936148:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1787828540.936156:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c64992c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085628032:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1787828540.936156:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828540.936168:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828540.936208:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a1c0 x1874672085628288/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828540.936227:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828540.936233:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787828540.936236:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828540.936261:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1976 00000100:00080000:3.0:1787828540.936272:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1787828540.936274:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1787828540.936290:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828540.936304:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:3.0:1787828540.936334:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c64983c0 x1874672085628416/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828540.936343:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828540/real 1787828540] req@ffff9e35c64992c0 x1874672085628032/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828565 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828540.936384:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828540.936391:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787828540.936399:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787828540.936401:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c64992c0 x1874672085628032/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828565 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828540.936439:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1976 00000100:00080000:3.0:1787828540.936455:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828540.936458:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828540.936483:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:1.0:1787828540.936490:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000008:00100000:3.0:1787828540.936495:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:1.0:1787828540.936499:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c64992c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085628032:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1787828540.936513:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085628160:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787828540.936519:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499e00 x1874672085628544/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828540.936532:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828540.936537:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828540.936540:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828540.936544:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1976 00000800:00000100:1.0:1787828540.936553:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1787828540.936556:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828540.936565:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787828540.936574:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828540/real 1787828540] req@ffff9e35c6498000 x1874672085628160/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828565 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1787828540.936575:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828540.936598:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498000 x1874672085628160/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828565 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828540.936607:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b480 x1874672085628672/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828540.936613:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828540.936625:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828540.936634:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085628160:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1787828540.936656:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828540.936661:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787828540.936663:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828540.936669:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1976 00000100:00080000:3.0:1787828540.936675:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1787828540.936676:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085628288:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1787828540.936685:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828540.936694:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1787828540.936710:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828540.936715:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b0c0 x1874672085628800/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828540.936729:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828540/real 1787828540] req@ffff9e35c649a1c0 x1874672085628288/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828565 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828540.936731:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828540.936744:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787828540.936749:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828540.936759:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1976 00000100:00080000:3.0:1787828540.936763:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828540.936767:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828540.936772:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:1.0:1787828540.936775:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a1c0 x1874672085628288/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828565 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828540.936787:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649ad00 x1874672085628928/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828540.936791:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828540.936839:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828540.936851:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085628288:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1787828540.936854:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828540.936863:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:1.0:1787828540.936866:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c64983c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085628416:192.168.203.141@tcp:8:kworker.0 00000100:00080000:3.0:1787828540.936868:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828540.936880:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1976 00000100:00080000:3.0:1787828540.936890:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828540.936898:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:1.0:1787828540.936907:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:3.0:1787828540.936908:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:1.0:1787828540.936935:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828540/real 1787828540] req@ffff9e35c64983c0 x1874672085628416/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828565 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828540.936937:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499a40 x1874672085629056/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828540.937015:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c64983c0 x1874672085628416/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828565 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828540.937029:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828540.937033:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828540.937037:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c64983c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085628416:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828540.937043:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085628544:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828540.937066:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828540.937078:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828540/real 1787828540] req@ffff9e35c6499e00 x1874672085628544/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828565 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828540.937089:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499e00 x1874672085628544/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828565 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828540.937102:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828540.937106:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828540.937111:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085628544:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828540.937119:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085628672:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787828540.937149:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828540.937166:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828540/real 1787828540] req@ffff9e35c649b480 x1874672085628672/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828565 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828540.937196:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b480 x1874672085628672/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828565 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828540.937204:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828540.937208:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828540.937211:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085628672:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828540.937231:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085628800:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787828540.937274:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828540.937288:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828540/real 1787828540] req@ffff9e35c649b0c0 x1874672085628800/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828565 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828540.937303:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b0c0 x1874672085628800/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828565 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828540.937314:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828540.937318:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828540.937322:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085628800:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828540.937328:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085628928:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828540.937346:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828540.937357:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828540/real 1787828540] req@ffff9e35c649ad00 x1874672085628928/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828565 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828540.937370:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649ad00 x1874672085628928/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828565 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828540.937381:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828540.937384:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828540.937388:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085628928:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828540.937395:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085629056:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828540.937445:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828540.937480:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828540/real 1787828540] req@ffff9e35c6499a40 x1874672085629056/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828565 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828540.937493:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499a40 x1874672085629056/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828565 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828540.937502:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828540.937505:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828540.937509:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085629056:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787828541.063963:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828541.064009:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787828541.064043:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828541.064074:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828542.087898:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787828542.087901:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828542.087930:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787828542.087941:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828543.111992:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828543.112003:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787828543.112051:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828543.112114:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828544.135906:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787828544.135923:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828544.135923:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828544.136018:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828545.160012:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828545.160014:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787828545.160045:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828545.160062:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828546.055952:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828546.055965:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828546.055970:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828546.055985:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1981 00000100:00080000:0.0:1787828546.056000:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828546.056019:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787828546.056078:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718000 x1874672085629184/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828546.056111:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828546.056116:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828546.056118:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828546.056122:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1981 00000100:00080000:0.0:1787828546.056125:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828546.056129:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828546.056142:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828546.056158:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718780 x1874672085629312/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828546.056168:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828546.056171:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787828546.056173:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828546.056177:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1981 00000100:00080000:0.0:1787828546.056179:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828546.056182:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828546.056186:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828546.056195:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da719a40 x1874672085629440/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828546.056204:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085629184:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1787828546.056208:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828546.056218:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787828546.056221:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828546.056226:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1981 00000100:00080000:0.0:1787828546.056228:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828546.056231:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828546.056235:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:0.0:1787828546.056247:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a580 x1874672085629568/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828546.056255:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828546.056259:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787828546.056261:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828546.056266:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1981 00000100:00080000:0.0:1787828546.056270:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828546.056273:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828546.056278:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:0.0:1787828546.056287:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7183c0 x1874672085629696/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828546.056295:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828546.056299:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828546.056301:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828546.056305:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1981 00000100:00080000:0.0:1787828546.056308:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828546.056311:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828546.056314:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1787828546.056324:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828546.056326:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a1c0 x1874672085629824/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828546.056338:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828546.056343:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787828546.056346:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828546.056352:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1981 00000100:00080000:0.0:1787828546.056368:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828546.056375:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828546.056394:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828546.056414:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a940 x1874672085629952/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828546.056422:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828546/real 1787828546] req@ffff9e35da718000 x1874672085629184/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828571 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828546.056437:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828546.056446:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787828546.056450:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828546.056459:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1981 00000100:00080000:0.0:1787828546.056464:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828546.056469:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828546.056475:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:1.0:1787828546.056478:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718000 x1874672085629184/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828571 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828546.056491:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718f00 x1874672085630080/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828546.056506:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828546.056506:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828546.056516:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787828546.056518:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828546.056522:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:0.0:1787828546.056526:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1981 00000100:00080000:0.0:1787828546.056531:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828546.056537:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787828546.056539:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085629184:192.168.203.141@tcp:250:kworker.0 00000008:00100000:0.0:1787828546.056542:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:0.0:1787828546.056557:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718b40 x1874672085630208/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828546.056563:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085629312:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787828546.056610:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828546.056631:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828546/real 1787828546] req@ffff9e35da718780 x1874672085629312/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828571 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828546.056651:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718780 x1874672085629312/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828571 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828546.056666:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828546.056672:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828546.056678:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085629312:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828546.056717:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085629440:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787828546.056745:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828546.056761:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828546/real 1787828546] req@ffff9e35da719a40 x1874672085629440/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828571 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828546.056780:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672085629440/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828571 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828546.056792:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828546.056842:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828546.056849:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085629440:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828546.056859:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085629568:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828546.056891:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828546.056908:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828546/real 1787828546] req@ffff9e35da71a580 x1874672085629568/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828571 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828546.056926:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672085629568/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828571 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828546.056938:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828546.056943:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828546.056949:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085629568:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828546.056959:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085629696:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828546.056990:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828546.057006:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828546/real 1787828546] req@ffff9e35da7183c0 x1874672085629696/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828571 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828546.057025:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672085629696/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828571 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828546.057038:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828546.057042:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828546.057047:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085629696:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828546.057057:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085629824:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787828546.057086:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828546.057101:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828546/real 1787828546] req@ffff9e35da71a1c0 x1874672085629824/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828571 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828546.057120:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a1c0 x1874672085629824/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828571 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828546.057132:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828546.057137:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828546.057142:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085629824:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828546.057150:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085629952:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787828546.057178:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828546.057195:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828546/real 1787828546] req@ffff9e35da71a940 x1874672085629952/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828571 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828546.057224:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672085629952/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828571 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828546.057237:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828546.057242:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828546.057247:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085629952:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828546.057256:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085630080:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828546.057283:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828546.057299:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828546/real 1787828546] req@ffff9e35da718f00 x1874672085630080/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828571 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828546.057317:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672085630080/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828571 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828546.057330:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828546.057335:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828546.057340:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085630080:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828546.057348:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085630208:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828546.057396:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828546.057414:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828546/real 1787828546] req@ffff9e35da718b40 x1874672085630208/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828571 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828546.057432:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718b40 x1874672085630208/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828571 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828546.057453:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828546.057458:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828546.057463:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085630208:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828546.183944:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787828546.183992:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787828546.184003:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828546.184099:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828547.207928:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787828547.207947:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828547.207971:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828547.208081:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828548.231920:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828548.231920:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787828548.231926:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828548.232038:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828549.255899:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828549.255903:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787828549.255954:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828549.255988:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828550.279916:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828550.279917:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828550.279952:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828550.279957:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787828551.176047:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828551.176063:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828551.176069:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828551.176083:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1986 00000100:00080000:1.0:1787828551.176116:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828551.176127:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787828551.176188:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f43c0 x1874672085630336/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828551.176238:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828551.176243:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828551.176246:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828551.176251:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1986 00000100:00080000:1.0:1787828551.176257:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828551.176262:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828551.176277:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828551.176293:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4000 x1874672085630464/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828551.176308:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828551.176312:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787828551.176315:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828551.176319:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1986 00000100:00080000:1.0:1787828551.176323:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828551.176329:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828551.176335:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828551.176378:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7480 x1874672085630592/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828551.176402:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828551.176407:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828551.176413:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787828551.176418:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085630336:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787828551.176423:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1986 00000100:00080000:1.0:1787828551.176435:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828551.176444:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828551.176452:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:1.0:1787828551.176470:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5e00 x1874672085630720/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828551.176483:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828551.176487:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828551.176490:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828551.176495:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1986 00000100:00080000:1.0:1787828551.176500:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828551.176504:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828551.176510:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:1.0:1787828551.176523:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6d00 x1874672085630848/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828551.176536:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828551.176540:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828551.176543:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828551.176549:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1986 00000100:00080000:1.0:1787828551.176557:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1787828551.176563:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787828551.176588:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828551.176600:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828551.176625:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4b40 x1874672085630976/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828551.176642:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828551.176649:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:0.0:1787828551.176653:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828551/real 1787828551] req@ffff9e35da6f43c0 x1874672085630336/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828576 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828551.176655:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828551.176669:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1986 00000100:00080000:1.0:1787828551.176679:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828551.176685:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787828551.176692:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f43c0 x1874672085630336/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828576 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1787828551.176695:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1787828551.176714:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828551.176720:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828551.176727:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4f00 x1874672085631104/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787828551.176728:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085630336:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1787828551.176746:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085630464:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787828551.176780:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787828551.176784:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828551.176789:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828551.176793:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787828551.176799:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828551/real 1787828551] req@ffff9e35da6f4000 x1874672085630464/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828576 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828551.176839:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1986 00000100:00080000:1.0:1787828551.176846:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787828551.176855:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4000 x1874672085630464/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828576 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828551.176858:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828551.176875:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00080000:0.0:1787828551.176877:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828551.176887:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828551.176892:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085630464:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828551.176909:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7840 x1874672085631232/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787828551.176927:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085630592:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787828551.176933:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828551.176940:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828551.176944:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828551.176952:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1986 00000800:00000100:0.0:1787828551.176959:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787828551.176962:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828551.176972:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787828551.176979:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828551/real 1787828551] req@ffff9e35da6f7480 x1874672085630592/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828576 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1787828551.176985:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:0.0:1787828551.177009:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7480 x1874672085630592/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828576 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828551.177018:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4780 x1874672085631360/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828551.177022:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828551.177033:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828551.177044:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085630592:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787828551.177059:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085630720:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787828551.177105:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828551.177121:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828551/real 1787828551] req@ffff9e35da6f5e00 x1874672085630720/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828576 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828551.177136:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5e00 x1874672085630720/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828576 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828551.177146:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828551.177149:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828551.177153:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085630720:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787828551.177160:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085630848:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787828551.177179:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828551.177190:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828551/real 1787828551] req@ffff9e35da6f6d00 x1874672085630848/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828576 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828551.177204:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6d00 x1874672085630848/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828576 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828551.177233:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828551.177239:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828551.177246:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085630848:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787828551.177259:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085630976:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787828551.177297:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828551.177318:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828551/real 1787828551] req@ffff9e35da6f4b40 x1874672085630976/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828576 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828551.177368:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4b40 x1874672085630976/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828576 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828551.177388:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828551.177395:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828551.177401:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085630976:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787828551.177414:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085631104:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787828551.177458:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828551.177479:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828551/real 1787828551] req@ffff9e35da6f4f00 x1874672085631104/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828576 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828551.177495:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4f00 x1874672085631104/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828576 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828551.177505:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828551.177509:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828551.177514:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085631104:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787828551.177550:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085631232:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787828551.177598:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828551.177609:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828551/real 1787828551] req@ffff9e35da6f7840 x1874672085631232/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828576 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828551.177621:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7840 x1874672085631232/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828576 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828551.177629:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828551.177632:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828551.177635:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085631232:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787828551.177640:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085631360:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787828551.177655:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828551.177664:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828551/real 1787828551] req@ffff9e35da6f4780 x1874672085631360/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828576 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828551.177675:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4780 x1874672085631360/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828576 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828551.177683:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828551.177686:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828551.177689:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085631360:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828551.303998:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828551.304027:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828551.304146:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787828551.304194:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828552.327980:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787828552.328029:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828552.328102:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828552.328111:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828553.351963:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787828553.351970:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787828553.352017:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828553.352184:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828554.375929:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828554.376024:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787828554.376031:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828554.376123:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828555.399998:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787828555.400055:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787828555.400078:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828555.400127:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787828556.295968:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828556.295978:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787828556.295983:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828556.295994:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1991 00000100:00080000:3.0:1787828556.296005:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828556.296013:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787828556.296055:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a580 x1874672085631488/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828556.296095:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828556.296102:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828556.296106:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828556.296112:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1991 00000100:00080000:3.0:1787828556.296126:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828556.296133:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828556.296147:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828556.296186:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499680 x1874672085631616/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828556.296200:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828556.296204:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787828556.296207:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828556.296213:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1991 00000100:00100000:0.0:1787828556.296218:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085631488:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1787828556.296220:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828556.296232:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828556.296240:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828556.296257:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498780 x1874672085631744/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828556.296269:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828556.296272:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787828556.296274:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828556.296279:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1991 00000100:00080000:3.0:1787828556.296284:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828556.296288:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828556.296294:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:3.0:1787828556.296328:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498b40 x1874672085631872/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787828556.296335:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1787828556.296349:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828556.296354:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787828556.296356:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828556.296363:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1991 00000100:00080000:3.0:1787828556.296368:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828556.296372:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828556.296379:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:0.0:1787828556.296394:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828556/real 1787828556] req@ffff9e35c649a580 x1874672085631488/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828581 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828556.296397:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a940 x1874672085632000/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828556.296442:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828556.296448:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828556.296452:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787828556.296459:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a580 x1874672085631488/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828581 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828556.296483:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1991 00000100:00080000:3.0:1787828556.296495:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828556.296509:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:0.0:1787828556.296515:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:3.0:1787828556.296519:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1787828556.296529:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828556.296540:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085631488:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1787828556.296551:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649bc00 x1874672085632128/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787828556.296563:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085631616:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1787828556.296583:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828556.296587:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787828556.296591:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828556.296614:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1991 00000100:00080000:3.0:1787828556.296620:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1787828556.296629:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1787828556.296629:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828556.296641:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828556.296657:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828556/real 1787828556] req@ffff9e35c6499680 x1874672085631616/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828581 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828556.296662:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b840 x1874672085632256/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828556.296715:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787828556.296719:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499680 x1874672085631616/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828581 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828556.296721:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787828556.296726:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828556.296740:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1991 00000100:00080000:0.0:1787828556.296745:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828556.296747:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828556.296757:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:3.0:1787828556.296758:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828556.296787:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:0.0:1787828556.296792:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085631616:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787828556.296813:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498f00 x1874672085632384/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828556.296830:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828556.296836:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787828556.296839:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787828556.296847:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085631744:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1787828556.296849:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1991 00000100:00080000:3.0:1787828556.296917:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828556.296925:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828556.296933:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:3.0:1787828556.296954:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499680 x1874672085632512/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787828556.296967:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828556.296989:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828556/real 1787828556] req@ffff9e35c6498780 x1874672085631744/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828581 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828556.297003:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498780 x1874672085631744/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828581 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828556.297012:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828556.297015:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828556.297019:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085631744:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787828556.297026:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085631872:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787828556.297043:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828556.297052:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828556/real 1787828556] req@ffff9e35c6498b40 x1874672085631872/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828581 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828556.297064:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498b40 x1874672085631872/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828581 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828556.297072:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828556.297074:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828556.297077:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085631872:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787828556.297083:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085632000:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787828556.297097:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828556.297106:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828556/real 1787828556] req@ffff9e35c649a940 x1874672085632000/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828581 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828556.297125:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a940 x1874672085632000/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828581 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828556.297134:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828556.297137:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828556.297140:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085632000:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787828556.297145:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085632128:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787828556.297159:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828556.297168:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828556/real 1787828556] req@ffff9e35c649bc00 x1874672085632128/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828581 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828556.297179:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649bc00 x1874672085632128/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828581 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828556.297187:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828556.297190:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828556.297193:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085632128:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787828556.297199:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085632256:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787828556.297213:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828556.297222:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828556/real 1787828556] req@ffff9e35c649b840 x1874672085632256/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828581 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828556.297233:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b840 x1874672085632256/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828581 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828556.297240:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828556.297243:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828556.297247:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085632256:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787828556.297252:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085632384:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787828556.297266:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828556.297275:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828556/real 1787828556] req@ffff9e35c6498f00 x1874672085632384/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828581 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828556.297285:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498f00 x1874672085632384/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828581 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828556.297293:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828556.297296:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828556.297299:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085632384:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787828556.297347:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085632512:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787828556.297369:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828556.297381:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828556/real 1787828556] req@ffff9e35c6499680 x1874672085632512/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828581 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828556.297392:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499680 x1874672085632512/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828581 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828556.297401:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828556.297404:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828556.297407:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085632512:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787828556.423930:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828556.423934:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828556.423938:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828556.424044:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828557.447866:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828557.447866:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828557.447873:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828557.447918:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828558.471914:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828558.471952:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828558.471969:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:3.0:1787828558.471981:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828559.495886:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:3.0:1787828559.495894:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828559.495923:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828559.495976:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828560.519893:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787828560.519908:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787828560.519938:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787828560.519943:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828561.415917:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828561.415927:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828561.415947:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828561.415955:0:27606:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1996 00000100:00080000:0.0:1787828561.415965:0:27606:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828561.415970:0:27606:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787828561.416002:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7192c0 x1874672085632640/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828561.416031:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828561.416034:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828561.416035:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828561.416038:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1996 00000100:00080000:0.0:1787828561.416040:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828561.416043:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828561.416050:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828561.416081:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718b40 x1874672085632768/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828561.416092:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828561.416098:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:1.0:1787828561.416099:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085632640:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1787828561.416103:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828561.416113:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1996 00000100:00080000:0.0:1787828561.416116:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828561.416119:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828561.416124:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828561.416144:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718f00 x1874672085632896/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828561.416157:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828561.416162:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787828561.416165:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828561.416173:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1996 00000800:00000100:1.0:1787828561.416176:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787828561.416182:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828561.416192:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828561.416199:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:0.0:1787828561.416213:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a940 x1874672085633024/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828561.416222:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828561/real 1787828561] req@ffff9e35da7192c0 x1874672085632640/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828586 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828561.416241:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828561.416250:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787828561.416254:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828561.416261:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 1996 00000100:00100000:1.0:1787828561.416266:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7192c0 x1874672085632640/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828586 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828561.416267:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828561.416287:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828561.416298:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00080000:1.0:1787828561.416299:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828561.416306:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828561.416310:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085632640:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1787828561.416316:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a1c0 x1874672085633152/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828561.416324:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828561.416327:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828561.416329:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787828561.416334:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085632768:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787828561.416335:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1996 00000100:00080000:0.0:1787828561.416341:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828561.416345:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828561.416350:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828561.416360:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7183c0 x1874672085633280/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787828561.416367:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787828561.416375:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828561.416394:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787828561.416397:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828561.416403:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1996 00000100:00100000:1.0:1787828561.416404:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828561/real 1787828561] req@ffff9e35da718b40 x1874672085632768/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828586 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828561.416407:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828561.416419:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828561.416428:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828561.416429:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718b40 x1874672085632768/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828586 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828561.416450:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1787828561.416451:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a580 x1874672085633408/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828561.416463:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828561.416475:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085632768:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787828561.416480:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828561.416487:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787828561.416489:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787828561.416495:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085632896:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787828561.416498:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1996 00000100:00080000:0.0:1787828561.416522:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828561.416527:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828561.416531:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:0.0:1787828561.416543:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da719a40 x1874672085633536/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828561.416550:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1787828561.416552:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787828561.416555:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787828561.416558:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828561.416562:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 1996 00000100:00080000:0.0:1787828561.416566:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1787828561.416568:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828561/real 1787828561] req@ffff9e35da718f00 x1874672085632896/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828586 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828561.416571:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828561.416578:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:1.0:1787828561.416587:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672085632896/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828586 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828561.416591:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718780 x1874672085633664/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828561.416610:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828561.416616:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828561.416621:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085632896:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828561.416629:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085633024:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828561.416651:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828561.416679:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828561/real 1787828561] req@ffff9e35da71a940 x1874672085633024/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828586 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828561.416695:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672085633024/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828586 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828561.416702:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828561.416705:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828561.416708:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085633024:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828561.416713:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085633152:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828561.416732:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828561.416741:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828561/real 1787828561] req@ffff9e35da71a1c0 x1874672085633152/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828586 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828561.416750:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a1c0 x1874672085633152/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828586 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828561.416758:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828561.416761:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828561.416763:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085633152:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828561.416780:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085633280:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787828561.416794:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828561.416802:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828561/real 1787828561] req@ffff9e35da7183c0 x1874672085633280/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828586 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828561.416842:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672085633280/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828586 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828561.416851:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828561.416854:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828561.416856:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085633280:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828561.416862:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085633408:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787828561.416878:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828561.416888:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828561/real 1787828561] req@ffff9e35da71a580 x1874672085633408/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828586 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828561.416898:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672085633408/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828586 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828561.416904:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828561.416907:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828561.416910:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085633408:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828561.416915:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085633536:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828561.416944:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828561.416950:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828561/real 1787828561] req@ffff9e35da719a40 x1874672085633536/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828586 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828561.416957:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672085633536/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828586 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828561.416962:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828561.416964:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828561.416966:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085633536:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828561.416969:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085633664:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828561.416979:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828561.416985:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828561/real 1787828561] req@ffff9e35da718780 x1874672085633664/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828586 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828561.416992:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718780 x1874672085633664/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828586 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828561.416997:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828561.416999:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828561.417000:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085633664:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828561.543859:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787828561.543869:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787828561.543880:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828561.543921:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828562.567884:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787828562.567888:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828562.567901:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828562.567919:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828563.591902:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828563.591919:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787828563.591949:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828563.591951:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828564.615888:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787828564.615891:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828564.615953:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828564.615972:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828565.639915:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828565.639929:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787828565.640081:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828565.640124:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828566.535962:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828566.535979:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828566.535987:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828566.536006:0:27606:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2001 00000100:00080000:1.0:1787828566.536023:0:27606:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828566.536051:0:27606:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787828566.536103:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f70c0 x1874672085633792/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828566.536157:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828566.536167:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828566.536171:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828566.536181:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2001 00000100:00080000:1.0:1787828566.536191:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828566.536201:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787828566.536217:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085633792:192.168.203.141@tcp:250:kworker.0 00000008:00100000:1.0:1787828566.536219:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828566.536283:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6940 x1874672085633920/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828566.536310:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828566.536318:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787828566.536323:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828566.536336:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2001 00000800:00000100:3.0:1787828566.536341:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787828566.536350:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828566.536362:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828566.536374:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828566.536394:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6580 x1874672085634048/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828566.536410:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828566.536415:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828566.536419:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828566.536425:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2001 00000100:00080000:1.0:1787828566.536431:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828566.536437:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828566.536445:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:1.0:1787828566.536456:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5a40 x1874672085634176/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828566.536468:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828566.536473:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828566.536476:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828566.536482:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2001 00000100:00080000:1.0:1787828566.536488:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828566.536495:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828566.536504:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:1.0:1787828566.536525:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f61c0 x1874672085634304/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828566.536547:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828566.536553:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828566.536558:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1787828566.536563:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828566/real 1787828566] req@ffff9e35da6f70c0 x1874672085633792/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828591 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828566.536572:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2001 00000100:00080000:1.0:1787828566.536582:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828566.536590:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828566.536600:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828566.536625:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4780 x1874672085634432/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828566.536638:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828566.536642:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787828566.536645:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828566.536650:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2001 00000100:00080000:1.0:1787828566.536654:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828566.536659:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828566.536665:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828566.536680:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7840 x1874672085634560/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828566.536691:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828566.536695:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828566.536698:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828566.536703:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2001 00000100:00080000:1.0:1787828566.536707:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828566.536711:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828566.536716:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:1.0:1787828566.536729:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4f00 x1874672085634688/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828566.536743:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828566.536751:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828566.536760:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828566.536774:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2001 00000100:00080000:1.0:1787828566.536784:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1787828566.536790:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f70c0 x1874672085633792/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828591 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828566.536835:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828566.536847:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:1.0:1787828566.536874:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4b40 x1874672085634816/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828566.536929:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828566.536939:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828566.536948:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085633792:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1787828566.536981:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085633920:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787828566.537020:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828566.537040:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828566/real 1787828566] req@ffff9e35da6f6940 x1874672085633920/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828591 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828566.537058:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6940 x1874672085633920/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828591 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828566.537070:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828566.537074:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828566.537078:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085633920:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787828566.537086:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085634048:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787828566.537107:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828566.537119:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828566/real 1787828566] req@ffff9e35da6f6580 x1874672085634048/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828591 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828566.537133:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6580 x1874672085634048/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828591 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828566.537143:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828566.537147:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828566.537150:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085634048:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787828566.537157:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085634176:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787828566.537177:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828566.537188:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828566/real 1787828566] req@ffff9e35da6f5a40 x1874672085634176/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828591 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828566.537202:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5a40 x1874672085634176/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828591 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828566.537221:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828566.537225:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828566.537229:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085634176:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787828566.537266:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085634304:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787828566.537298:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828566.537313:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828566/real 1787828566] req@ffff9e35da6f61c0 x1874672085634304/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828591 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828566.537327:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f61c0 x1874672085634304/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828591 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828566.537339:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828566.537343:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828566.537346:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085634304:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787828566.537353:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085634432:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787828566.537374:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828566.537392:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828566/real 1787828566] req@ffff9e35da6f4780 x1874672085634432/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828591 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828566.537415:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4780 x1874672085634432/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828591 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828566.537433:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828566.537439:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828566.537447:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085634432:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787828566.537459:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085634560:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787828566.537500:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828566.537518:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828566/real 1787828566] req@ffff9e35da6f7840 x1874672085634560/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828591 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828566.537533:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7840 x1874672085634560/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828591 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828566.537543:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828566.537547:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828566.537551:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085634560:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787828566.537559:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085634688:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787828566.537581:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828566.537593:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828566/real 1787828566] req@ffff9e35da6f4f00 x1874672085634688/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828591 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828566.537606:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4f00 x1874672085634688/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828591 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828566.537616:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828566.537620:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828566.537624:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085634688:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787828566.537631:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085634816:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787828566.537649:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828566.537661:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828566/real 1787828566] req@ffff9e35da6f4b40 x1874672085634816/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828591 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828566.537674:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4b40 x1874672085634816/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828591 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828566.537684:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828566.537687:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828566.537691:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085634816:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787828566.663937:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828566.663945:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828566.663972:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:0.0:1787828566.663994:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828567.687918:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828567.687942:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828567.687978:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828567.687986:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828568.711899:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787828568.711913:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787828568.711957:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828568.711974:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828569.735958:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787828569.735970:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787828569.735984:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787828569.736174:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828570.759979:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787828570.759987:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787828570.760031:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828570.760043:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828571.656070:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828571.656081:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787828571.656086:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828571.656096:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2007 00000100:00080000:3.0:1787828571.656109:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828571.656116:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787828571.656156:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a580 x1874672085634944/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828571.656190:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828571.656194:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828571.656196:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828571.656229:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2007 00000100:00080000:3.0:1787828571.656233:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828571.656254:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828571.656266:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828571.656275:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085634944:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1787828571.656290:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499a40 x1874672085635072/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828571.656304:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828571.656307:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787828571.656309:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828571.656314:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2007 00000100:00080000:3.0:1787828571.656319:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828571.656322:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828571.656339:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828571.656351:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649ad00 x1874672085635200/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828571.656365:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828571.656372:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000100:1.0:1787828571.656373:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1787828571.656376:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828571.656383:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2007 00000100:00080000:3.0:1787828571.656388:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828571.656392:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828571.656398:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:3.0:1787828571.656410:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b0c0 x1874672085635328/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828571.656419:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787828571.656425:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828571/real 1787828571] req@ffff9e35c649a580 x1874672085634944/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828596 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828571.656426:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787828571.656431:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828571.656441:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2007 00000100:00080000:3.0:1787828571.656450:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828571.656454:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828571.656459:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:1.0:1787828571.656460:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a580 x1874672085634944/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828596 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828571.656475:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1787828571.656477:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b480 x1874672085635456/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828571.656483:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828571.656495:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085634944:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1787828571.656499:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828571.656504:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828571.656506:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828571.656512:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2007 00000100:00080000:3.0:1787828571.656514:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828571.656518:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828571.656523:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828571.656524:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085635072:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787828571.656537:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499e00 x1874672085635584/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828571.656550:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1787828571.656556:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1787828571.656558:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787828571.656562:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828571.656568:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2007 00000100:00080000:3.0:1787828571.656573:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1787828571.656577:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828571/real 1787828571] req@ffff9e35c6499a40 x1874672085635072/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828596 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828571.656582:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828571.656591:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828571.656603:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499a40 x1874672085635072/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828596 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828571.656610:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c64983c0 x1874672085635712/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828571.656625:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828571.656630:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828571.656634:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828571.656638:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085635072:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1787828571.656647:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787828571.656649:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828571.656654:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2007 00000100:00080000:3.0:1787828571.656661:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1787828571.656666:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085635200:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1787828571.656668:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828571.656678:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:3.0:1787828571.656696:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a1c0 x1874672085635840/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787828571.656702:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1787828571.656738:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787828571.656744:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828571/real 1787828571] req@ffff9e35c649ad00 x1874672085635200/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828596 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828571.656745:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787828571.656750:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828571.656760:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2007 00000100:00080000:3.0:1787828571.656770:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1787828571.656776:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649ad00 x1874672085635200/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828596 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828571.656777:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828571.656788:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00080000:1.0:1787828571.656794:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828571.656801:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828571.656810:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085635200:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787828571.656811:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498000 x1874672085635968/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828571.656857:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085635328:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828571.656884:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828571.656901:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828571/real 1787828571] req@ffff9e35c649b0c0 x1874672085635328/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828596 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828571.656914:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b0c0 x1874672085635328/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828596 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828571.656923:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828571.656926:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828571.656929:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085635328:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828571.656934:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085635456:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828571.656949:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828571.656974:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828571/real 1787828571] req@ffff9e35c649b480 x1874672085635456/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828596 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828571.656985:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b480 x1874672085635456/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828596 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828571.656994:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828571.656996:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828571.656999:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085635456:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828571.657022:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085635584:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787828571.657038:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828571.657047:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828571/real 1787828571] req@ffff9e35c6499e00 x1874672085635584/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828596 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828571.657075:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499e00 x1874672085635584/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828596 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828571.657084:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828571.657086:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828571.657089:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085635584:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828571.657094:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c64983c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085635712:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787828571.657109:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828571.657117:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828571/real 1787828571] req@ffff9e35c64983c0 x1874672085635712/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828596 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828571.657127:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c64983c0 x1874672085635712/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828596 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828571.657135:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828571.657138:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828571.657140:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c64983c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085635712:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828571.657145:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085635840:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828571.657158:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828571.657166:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828571/real 1787828571] req@ffff9e35c649a1c0 x1874672085635840/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828596 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828571.657176:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a1c0 x1874672085635840/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828596 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828571.657183:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828571.657186:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828571.657190:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085635840:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828571.657195:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085635968:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828571.657255:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828571.657271:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828571/real 1787828571] req@ffff9e35c6498000 x1874672085635968/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828596 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828571.657285:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498000 x1874672085635968/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828596 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828571.657294:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828571.657297:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828571.657310:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085635968:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828571.783983:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787828571.784003:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828571.784077:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828571.784094:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828572.807914:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828572.807914:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787828572.807970:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787828572.807982:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828573.831902:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787828573.831902:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828573.831912:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828573.831993:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828574.855909:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828574.855909:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828574.855943:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828574.855948:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828575.879901:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828575.879902:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787828575.879940:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787828575.880012:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828576.775872:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828576.775880:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787828576.775884:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828576.775892:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2012 00000100:00080000:3.0:1787828576.775902:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828576.775909:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787828576.775944:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c64992c0 x1874672085636096/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828576.775972:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828576.775976:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828576.775977:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828576.775980:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2012 00000100:00080000:3.0:1787828576.775983:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828576.775986:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828576.775994:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828576.776006:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498000 x1874672085636224/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828576.776014:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828576.776019:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:1.0:1787828576.776020:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c64992c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085636096:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1787828576.776021:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828576.776028:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2012 00000100:00080000:3.0:1787828576.776032:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828576.776034:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828576.776037:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828576.776046:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a1c0 x1874672085636352/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828576.776054:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828576.776056:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787828576.776057:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828576.776061:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2012 00000100:00080000:3.0:1787828576.776063:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828576.776065:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828576.776068:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:3.0:1787828576.776076:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c64983c0 x1874672085636480/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787828576.776083:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1787828576.776090:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828576.776093:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787828576.776095:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828576.776099:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2012 00000100:00080000:3.0:1787828576.776101:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828576.776104:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828576.776106:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:3.0:1787828576.776114:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499e00 x1874672085636608/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828576.776121:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787828576.776123:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828576/real 1787828576] req@ffff9e35c64992c0 x1874672085636096/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828601 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828576.776124:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828576.776128:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828576.776135:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2012 00000100:00080000:3.0:1787828576.776140:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828576.776144:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828576.776148:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828576.776151:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c64992c0 x1874672085636096/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828601 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828576.776158:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b480 x1874672085636736/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828576.776179:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828576.776199:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:3.0:1787828576.776199:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828576.776204:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:1.0:1787828576.776205:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c64992c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085636096:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1787828576.776207:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828576.776215:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2012 00000100:00080000:3.0:1787828576.776218:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828576.776221:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828576.776225:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828576.776233:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085636224:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787828576.776237:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b0c0 x1874672085636864/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828576.776245:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828576.776248:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787828576.776250:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828576.776253:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2012 00000800:00000100:1.0:1787828576.776264:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1787828576.776269:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828576.776274:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828576.776278:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:1.0:1787828576.776281:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828576/real 1787828576] req@ffff9e35c6498000 x1874672085636224/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828601 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828576.776289:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649ad00 x1874672085636992/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828576.776295:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498000 x1874672085636224/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828601 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828576.776328:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828576.776329:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828576.776335:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828576.776336:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:3.0:1787828576.776339:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787828576.776342:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085636224:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1787828576.776346:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2012 00000100:00080000:3.0:1787828576.776351:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1787828576.776353:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085636352:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1787828576.776357:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828576.776362:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:3.0:1787828576.776372:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499a40 x1874672085637120/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787828576.776377:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828576.776390:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828576/real 1787828576] req@ffff9e35c649a1c0 x1874672085636352/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828601 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828576.776400:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a1c0 x1874672085636352/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828601 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828576.776406:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828576.776409:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828576.776411:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085636352:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828576.776415:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c64983c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085636480:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828576.776433:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828576.776440:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828576/real 1787828576] req@ffff9e35c64983c0 x1874672085636480/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828601 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828576.776447:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c64983c0 x1874672085636480/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828601 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828576.776452:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828576.776455:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828576.776457:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c64983c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085636480:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828576.776460:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085636608:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828576.776471:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828576.776476:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828576/real 1787828576] req@ffff9e35c6499e00 x1874672085636608/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828601 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828576.776483:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499e00 x1874672085636608/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828601 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828576.776489:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828576.776490:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828576.776492:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085636608:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828576.776496:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085636736:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787828576.776512:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828576.776518:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828576/real 1787828576] req@ffff9e35c649b480 x1874672085636736/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828601 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828576.776531:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b480 x1874672085636736/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828601 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828576.776545:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828576.776547:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828576.776549:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085636736:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828576.776561:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085636864:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787828576.776572:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828576.776578:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828576/real 1787828576] req@ffff9e35c649b0c0 x1874672085636864/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828601 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828576.776585:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b0c0 x1874672085636864/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828601 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828576.776590:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828576.776592:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828576.776594:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085636864:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828576.776597:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085636992:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828576.776606:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828576.776612:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828576/real 1787828576] req@ffff9e35c649ad00 x1874672085636992/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828601 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828576.776619:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649ad00 x1874672085636992/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828601 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828576.776628:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828576.776631:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828576.776633:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085636992:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828576.776636:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085637120:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828576.776644:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828576.776649:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828576/real 1787828576] req@ffff9e35c6499a40 x1874672085637120/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828601 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828576.776657:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499a40 x1874672085637120/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828601 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828576.776662:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828576.776664:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828576.776665:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085637120:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828576.903882:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787828576.903890:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828576.903916:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828576.903941:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828577.927947:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828577.927954:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787828577.927968:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828577.928059:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828578.952009:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787828578.952012:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828578.952060:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:0.0:1787828578.952109:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828579.975941:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828579.975956:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787828579.976009:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828579.976121:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828580.999912:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828580.999915:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787828580.999920:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828581.000008:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828581.895933:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828581.895943:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828581.895948:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828581.895974:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2017 00000100:00080000:0.0:1787828581.895985:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828581.895992:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787828581.896045:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718000 x1874672085637248/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828581.896071:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828581.896075:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828581.896077:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828581.896081:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2017 00000100:00080000:0.0:1787828581.896084:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828581.896087:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828581.896114:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828581.896170:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718780 x1874672085637376/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828581.896177:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085637248:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1787828581.896195:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828581.896200:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787828581.896202:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828581.896207:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2017 00000100:00080000:0.0:1787828581.896211:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828581.896216:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828581.896222:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828581.896235:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da719a40 x1874672085637504/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828581.896246:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828581.896250:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000100:1.0:1787828581.896253:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787828581.896255:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828581.896264:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2017 00000100:00080000:0.0:1787828581.896269:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828581.896274:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828581.896279:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:0.0:1787828581.896292:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a580 x1874672085637632/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828581.896302:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787828581.896306:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828581/real 1787828581] req@ffff9e35da718000 x1874672085637248/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828606 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828581.896307:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787828581.896312:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828581.896324:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2017 00000100:00080000:0.0:1787828581.896333:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828581.896337:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787828581.896344:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718000 x1874672085637248/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828606 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1787828581.896346:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00080000:1.0:1787828581.896364:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1787828581.896366:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7183c0 x1874672085637760/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828581.896375:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828581.896387:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085637248:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1787828581.896390:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828581.896395:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828581.896397:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828581.896402:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2017 00000100:00080000:0.0:1787828581.896405:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828581.896408:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828581.896412:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828581.896417:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085637376:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787828581.896425:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a1c0 x1874672085637888/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828581.896438:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828581.896442:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787828581.896444:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1787828581.896449:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787828581.896454:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2017 00000100:00080000:0.0:1787828581.896461:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828581.896465:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828581.896471:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828581.896473:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828581/real 1787828581] req@ffff9e35da718780 x1874672085637376/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828606 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828581.896484:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a940 x1874672085638016/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828581.896533:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828581.896539:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:1.0:1787828581.896544:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718780 x1874672085637376/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828606 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828581.896545:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828581.896559:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2017 00000100:00080000:0.0:1787828581.896570:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828581.896574:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828581.896582:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:1.0:1787828581.896589:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000008:00100000:0.0:1787828581.896594:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:1.0:1787828581.896599:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085637376:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828581.896613:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085637504:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787828581.896618:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718f00 x1874672085638144/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828581.896631:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828581.896634:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787828581.896637:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828581.896642:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2017 00000100:00080000:0.0:1787828581.896647:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1787828581.896651:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787828581.896657:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828581.896664:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:1.0:1787828581.896671:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828581/real 1787828581] req@ffff9e35da719a40 x1874672085637504/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828606 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828581.896681:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718b40 x1874672085638272/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828581.896692:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672085637504/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828606 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828581.896704:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828581.896707:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828581.896710:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085637504:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828581.896719:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085637632:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828581.896737:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828581.896748:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828581/real 1787828581] req@ffff9e35da71a580 x1874672085637632/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828606 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828581.896758:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672085637632/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828606 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828581.896766:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828581.896768:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828581.896770:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085637632:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828581.896776:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085637760:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828581.896789:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828581.896796:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828581/real 1787828581] req@ffff9e35da7183c0 x1874672085637760/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828606 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828581.896860:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672085637760/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828606 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828581.896875:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828581.896879:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828581.896884:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085637760:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828581.896906:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085637888:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787828581.896928:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828581.896940:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828581/real 1787828581] req@ffff9e35da71a1c0 x1874672085637888/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828606 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828581.896951:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a1c0 x1874672085637888/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828606 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828581.896959:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828581.896962:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828581.896964:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085637888:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828581.896969:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085638016:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787828581.896983:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828581.896990:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828581/real 1787828581] req@ffff9e35da71a940 x1874672085638016/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828606 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828581.897000:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672085638016/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828606 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828581.897007:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828581.897010:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828581.897012:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085638016:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828581.897017:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085638144:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828581.897029:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828581.897036:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828581/real 1787828581] req@ffff9e35da718f00 x1874672085638144/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828606 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828581.897046:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672085638144/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828606 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828581.897060:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828581.897062:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828581.897065:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085638144:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828581.897069:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085638272:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828581.897081:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828581.897089:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828581/real 1787828581] req@ffff9e35da718b40 x1874672085638272/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828606 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828581.897099:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718b40 x1874672085638272/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828606 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828581.897107:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828581.897109:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828581.897112:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085638272:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787828582.023890:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828582.023890:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828582.023904:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:0.0:1787828582.023994:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828583.047869:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787828583.047918:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828583.047928:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828583.047959:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828584.071942:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828584.071942:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828584.072007:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828584.072231:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828585.095949:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828585.095949:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828585.095984:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828585.096007:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828586.119873:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828586.119915:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828586.119935:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828586.119946:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828587.015905:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828587.015919:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787828587.015925:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828587.015947:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2022 00000100:00080000:3.0:1787828587.015961:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828587.015970:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787828587.016030:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a580 x1874672085638400/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828587.016077:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828587.016102:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828587.016106:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828587.016113:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2022 00000100:00080000:3.0:1787828587.016119:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828587.016128:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828587.016144:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828587.016175:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499680 x1874672085638528/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828587.016176:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085638400:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1787828587.016197:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828587.016201:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787828587.016203:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828587.016207:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2022 00000100:00080000:3.0:1787828587.016211:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828587.016215:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828587.016220:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828587.016232:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498f00 x1874672085638656/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828587.016242:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828587.016245:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787828587.016247:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828587.016262:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2022 00000100:00080000:3.0:1787828587.016266:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828587.016271:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828587.016275:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:3.0:1787828587.016286:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b840 x1874672085638784/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828587.016294:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828587.016297:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787828587.016299:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828587.016303:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2022 00000100:00080000:3.0:1787828587.016306:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828587.016309:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828587.016313:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:3.0:1787828587.016324:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649bc00 x1874672085638912/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828587.016332:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828587.016336:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828587.016338:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828587.016345:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2022 00000100:00080000:3.0:1787828587.016347:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828587.016350:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787828587.016353:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085638528:192.168.203.141@tcp:38:kworker.0 00000008:00100000:3.0:1787828587.016355:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828587.016374:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a940 x1874672085639040/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828587.016391:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787828587.016396:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085638656:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1787828587.016397:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787828587.016402:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828587.016410:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2022 00000100:00080000:3.0:1787828587.016415:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828587.016419:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828587.016424:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828587.016436:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085638784:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787828587.016439:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498b40 x1874672085639168/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828587.016450:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828587.016455:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787828587.016457:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787828587.016465:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085638912:192.168.203.141@tcp:8:kworker.0 00000100:00080000:3.0:1787828587.016465:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2022 00000100:00080000:3.0:1787828587.016469:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828587.016474:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828587.016479:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:3.0:1787828587.016492:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498780 x1874672085639296/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828587.016500:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085639040:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1787828587.016517:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828587.016522:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:1.0:1787828587.016526:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085639168:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1787828587.016529:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828587.016538:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2022 00000100:00080000:3.0:1787828587.016545:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828587.016550:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828587.016555:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:1.0:1787828587.016561:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085639296:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787828587.016571:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499a40 x1874672085639424/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828587.016593:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085639424:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787828587.143922:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828587.143922:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828587.143922:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787828587.143974:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828588.167898:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828588.167898:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828588.167915:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828588.167936:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828589.191916:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828589.191916:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828589.191934:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828589.192029:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1787828590.088184:0:2398:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.141:988 00000400:00000100:0.0:1787828590.088228:0:2398:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.141@tcp at host 192.168.203.141 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1787828590.088244:0:2398:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:0.0:1787828590.088288:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000800:00000100:0.0:1787828590.088300:0:2398:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:0.0:1787828590.088311:0:2398:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:0.0:1787828590.088321:0:2398:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:0.0:1787828590.088332:0:2398:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:0.0:1787828590.088345:0:2398:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:1.0:1787828590.088349:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828587/real 1787828590] req@ffff9e35c6499a40 x1874672085639424/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828612 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:0.0:1787828590.088355:0:2398:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:0.0:1787828590.088367:0:2398:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:1.0:1787828590.088402:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499a40 x1874672085639424/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828612 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828590.088431:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828590.088438:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828590.088445:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085639424:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828590.088456:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828587/real 1787828590] req@ffff9e35c6498780 x1874672085639296/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828612 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828590.088495:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498780 x1874672085639296/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828612 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828590.088517:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828590.088522:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828590.088527:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085639296:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828590.088537:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828587/real 1787828590] req@ffff9e35c649a940 x1874672085639040/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828612 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828590.088553:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a940 x1874672085639040/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828612 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828590.088565:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828590.088581:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828590.088585:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085639040:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828590.088593:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828587/real 1787828590] req@ffff9e35c6498b40 x1874672085639168/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828612 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828590.088607:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498b40 x1874672085639168/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828612 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828590.088619:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828590.088624:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828590.088628:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085639168:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828590.088637:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828587/real 1787828590] req@ffff9e35c6499680 x1874672085638528/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828612 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828590.088653:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499680 x1874672085638528/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828612 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828590.088666:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828590.088670:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828590.088676:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085638528:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828590.088685:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828587/real 1787828590] req@ffff9e35c6498f00 x1874672085638656/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828612 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828590.088702:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498f00 x1874672085638656/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828612 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828590.088733:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828590.088740:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828590.088745:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085638656:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828590.088756:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828587/real 1787828590] req@ffff9e35c649b840 x1874672085638784/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828612 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828590.088774:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b840 x1874672085638784/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828612 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828590.088783:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828590.088835:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828590.088842:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085638784:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828590.088853:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828587/real 1787828590] req@ffff9e35c649bc00 x1874672085638912/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828612 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828590.088873:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649bc00 x1874672085638912/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828612 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828590.088887:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828590.088892:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828590.088896:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085638912:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828590.088906:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828587/real 1787828590] req@ffff9e35c649a580 x1874672085638400/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828612 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828590.088925:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a580 x1874672085638400/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828612 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828590.088936:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828590.088941:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828590.088945:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085638400:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1787828590.215886:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787828590.215890:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828590.215933:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787828590.215998:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828591.239999:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828591.239999:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828591.239999:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828591.240076:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828592.135951:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828592.135960:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828592.135964:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828592.135974:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2027 00000100:00080000:0.0:1787828592.135986:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828592.135993:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787828592.136031:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7192c0 x1874672085639552/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828592.136067:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828592.136070:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828592.136072:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828592.136075:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2027 00000100:00080000:0.0:1787828592.136078:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828592.136081:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828592.136093:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828592.136116:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085639552:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1787828592.136118:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718b40 x1874672085639680/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828592.136135:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828592.136139:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787828592.136141:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828592.136146:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2027 00000100:00080000:0.0:1787828592.136150:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828592.136154:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828592.136159:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828592.136169:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718f00 x1874672085639808/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828592.136176:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828592.136179:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787828592.136180:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828592.136184:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2027 00000100:00080000:0.0:1787828592.136187:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828592.136189:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828592.136192:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000800:00000100:1.0:1787828592.136202:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828592.136205:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a940 x1874672085639936/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828592.136217:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828592.136222:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787828592.136230:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828592.136236:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2027 00000100:00080000:0.0:1787828592.136240:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828592.136245:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828592.136251:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:1.0:1787828592.136255:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828592/real 1787828592] req@ffff9e35da7192c0 x1874672085639552/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828617 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828592.136268:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a1c0 x1874672085640064/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828592.136280:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787828592.136282:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7192c0 x1874672085639552/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828617 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828592.136287:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828592.136291:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828592.136302:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2027 00000100:00080000:0.0:1787828592.136309:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828592.136309:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828592.136314:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828592.136320:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00080000:1.0:1787828592.136325:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828592.136333:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085639552:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1787828592.136337:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7183c0 x1874672085640192/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828592.136355:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828592.136359:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787828592.136362:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828592.136367:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2027 00000100:00080000:0.0:1787828592.136371:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828592.136376:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828592.136383:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828592.136383:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085639680:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787828592.136400:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a580 x1874672085640320/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828592.136412:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828592.136416:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787828592.136419:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828592.136424:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2027 00000800:00000100:1.0:1787828592.136430:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787828592.136431:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828592.136439:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828592.136445:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:0.0:1787828592.136462:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da719a40 x1874672085640448/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828592.136462:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828592/real 1787828592] req@ffff9e35da718b40 x1874672085639680/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828617 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828592.136489:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828592.136495:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787828592.136498:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787828592.136500:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718b40 x1874672085639680/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828617 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828592.136508:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2027 00000100:00080000:0.0:1787828592.136516:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828592.136521:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828592.136523:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:1.0:1787828592.136531:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000008:00100000:0.0:1787828592.136533:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:1.0:1787828592.136538:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085639680:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787828592.136552:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718780 x1874672085640576/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828592.136553:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085639808:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787828592.136589:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828592.136606:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828592/real 1787828592] req@ffff9e35da718f00 x1874672085639808/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828617 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828592.136620:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672085639808/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828617 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828592.136629:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828592.136634:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828592.136637:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085639808:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828592.136643:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085639936:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828592.136666:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828592.136678:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828592/real 1787828592] req@ffff9e35da71a940 x1874672085639936/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828617 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828592.136691:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672085639936/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828617 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828592.136700:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828592.136703:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828592.136707:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085639936:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828592.136713:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085640064:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828592.136733:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828592.136745:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828592/real 1787828592] req@ffff9e35da71a1c0 x1874672085640064/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828617 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828592.136758:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a1c0 x1874672085640064/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828617 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828592.136768:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828592.136779:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828592.136784:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085640064:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828592.136795:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085640192:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787828592.136868:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828592.136881:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828592/real 1787828592] req@ffff9e35da7183c0 x1874672085640192/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828617 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828592.136893:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672085640192/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828617 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828592.136909:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828592.136913:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828592.136916:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085640192:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828592.136941:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085640320:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787828592.136964:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828592.136978:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828592/real 1787828592] req@ffff9e35da71a580 x1874672085640320/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828617 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828592.136992:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672085640320/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828617 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828592.137002:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828592.137006:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828592.137009:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085640320:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828592.137015:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085640448:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828592.137037:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828592.137048:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828592/real 1787828592] req@ffff9e35da719a40 x1874672085640448/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828617 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828592.137193:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672085640448/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828617 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828592.137202:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828592.137205:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828592.137210:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085640448:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828592.137217:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085640576:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828592.137238:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828592.137251:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828592/real 1787828592] req@ffff9e35da718780 x1874672085640576/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828617 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828592.137264:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718780 x1874672085640576/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828617 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828592.137274:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828592.137278:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828592.137282:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085640576:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828592.263882:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828592.263883:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787828592.263905:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828592.263967:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828593.287986:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787828593.287988:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828593.287994:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787828593.288116:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828594.311929:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828594.311929:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787828594.311949:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828594.312012:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828595.335912:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828595.335912:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828595.335923:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787828595.335962:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828596.359918:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828596.359918:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787828596.359922:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787828596.360007:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828597.255945:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828597.255953:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787828597.255956:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828597.255964:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2032 00000100:00080000:3.0:1787828597.255973:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828597.255978:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787828597.256010:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649ad00 x1874672085640704/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828597.256050:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828597.256053:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828597.256054:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828597.256058:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2032 00000100:00080000:3.0:1787828597.256060:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828597.256063:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828597.256070:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828597.256084:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b0c0 x1874672085640832/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828597.256093:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828597.256096:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787828597.256099:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828597.256107:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2032 00000100:00100000:1.0:1787828597.256110:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085640704:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1787828597.256111:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828597.256122:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828597.256131:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828597.256140:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b480 x1874672085640960/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828597.256148:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828597.256151:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787828597.256153:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828597.256156:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2032 00000100:00080000:3.0:1787828597.256158:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828597.256160:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828597.256164:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:3.0:1787828597.256172:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499e00 x1874672085641088/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828597.256178:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828597.256181:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787828597.256182:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828597.256186:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2032 00000100:00080000:3.0:1787828597.256188:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828597.256191:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828597.256193:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:3.0:1787828597.256199:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c64983c0 x1874672085641216/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828597.256206:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828597.256209:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828597.256211:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828597.256213:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2032 00000100:00080000:3.0:1787828597.256216:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828597.256217:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:1.0:1787828597.256222:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:3.0:1787828597.256227:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828597.256239:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a1c0 x1874672085641344/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828597.256250:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828597.256252:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787828597.256253:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828597.256257:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2032 00000100:00080000:3.0:1787828597.256259:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828597.256261:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828597.256266:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828597.256274:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498000 x1874672085641472/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828597.256279:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828597/real 1787828597] req@ffff9e35c649ad00 x1874672085640704/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828622 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828597.256306:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828597.256317:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787828597.256318:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828597.256326:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2032 00000100:00080000:3.0:1787828597.256329:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1787828597.256330:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649ad00 x1874672085640704/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828622 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828597.256343:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828597.256358:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00080000:1.0:1787828597.256360:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828597.256369:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828597.256376:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c64992c0 x1874672085641600/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828597.256381:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085640704:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1787828597.256402:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828597.256405:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787828597.256407:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828597.256410:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2032 00000100:00080000:3.0:1787828597.256415:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828597.256417:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828597.256419:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:1.0:1787828597.256436:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085640832:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787828597.256436:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649ad00 x1874672085641728/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787828597.256485:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828597.256502:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828597/real 1787828597] req@ffff9e35c649b0c0 x1874672085640832/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828622 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828597.256512:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b0c0 x1874672085640832/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828622 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828597.256519:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828597.256522:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828597.256524:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085640832:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828597.256530:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085640960:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787828597.256552:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828597.256560:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828597/real 1787828597] req@ffff9e35c649b480 x1874672085640960/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828622 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828597.256567:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b480 x1874672085640960/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828622 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828597.256573:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828597.256576:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828597.256578:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085640960:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828597.256582:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085641088:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828597.256594:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828597.256601:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828597/real 1787828597] req@ffff9e35c6499e00 x1874672085641088/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828622 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828597.256609:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499e00 x1874672085641088/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828622 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828597.256615:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828597.256617:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828597.256618:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085641088:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828597.256624:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c64983c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085641216:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828597.256644:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828597.256655:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828597/real 1787828597] req@ffff9e35c64983c0 x1874672085641216/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828622 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828597.256668:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c64983c0 x1874672085641216/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828622 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828597.256678:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828597.256681:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828597.256684:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c64983c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085641216:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828597.256688:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085641344:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787828597.256705:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828597.256713:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828597/real 1787828597] req@ffff9e35c649a1c0 x1874672085641344/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828622 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828597.256720:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a1c0 x1874672085641344/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828622 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828597.256727:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828597.256729:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828597.256731:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085641344:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828597.256737:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085641472:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787828597.256755:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828597.256767:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828597/real 1787828597] req@ffff9e35c6498000 x1874672085641472/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828622 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828597.256780:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498000 x1874672085641472/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828622 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828597.256789:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828597.256792:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828597.256794:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085641472:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828597.256835:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c64992c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085641600:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828597.256853:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828597.256862:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828597/real 1787828597] req@ffff9e35c64992c0 x1874672085641600/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828622 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828597.256869:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c64992c0 x1874672085641600/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828622 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828597.256876:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828597.256878:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828597.256881:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c64992c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085641600:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828597.256903:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085641728:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828597.256914:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828597.256921:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828597/real 1787828597] req@ffff9e35c649ad00 x1874672085641728/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828622 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828597.256928:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649ad00 x1874672085641728/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828622 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828597.256934:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828597.256936:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828597.256938:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085641728:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787828597.383931:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828597.383931:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828597.383931:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787828597.383983:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828598.407881:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787828598.407937:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828598.407968:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828598.407994:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828599.431908:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828599.431908:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828599.431911:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787828599.431959:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828600.455901:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828600.455922:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828600.455934:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787828600.455996:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828601.479874:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787828601.479875:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828601.479916:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787828601.479923:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828602.375928:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828602.375938:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787828602.375943:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828602.375954:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2037 00000100:00080000:3.0:1787828602.375967:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828602.375975:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787828602.376033:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a580 x1874672085641856/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828602.376058:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828602.376062:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828602.376063:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828602.376066:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2037 00000100:00080000:3.0:1787828602.376069:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828602.376072:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828602.376083:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828602.376094:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649bc00 x1874672085641984/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828602.376103:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828602.376105:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787828602.376107:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828602.376111:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2037 00000100:00080000:3.0:1787828602.376114:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1787828602.376120:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085641856:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1787828602.376120:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828602.376126:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828602.376138:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b840 x1874672085642112/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828602.376146:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828602.376150:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787828602.376152:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828602.376155:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2037 00000100:00080000:3.0:1787828602.376159:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828602.376162:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828602.376166:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:3.0:1787828602.376175:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498f00 x1874672085642240/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828602.376183:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828602.376186:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787828602.376187:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828602.376191:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2037 00000100:00080000:3.0:1787828602.376195:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828602.376198:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828602.376201:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000800:00000100:1.0:1787828602.376208:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828602.376211:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499680 x1874672085642368/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828602.376219:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828602.376222:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828602.376224:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828602.376228:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2037 00000100:00080000:3.0:1787828602.376231:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828602.376234:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787828602.376262:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828602/real 1787828602] req@ffff9e35c649a580 x1874672085641856/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828627 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828602.376283:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a580 x1874672085641856/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828627 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828602.376298:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828602.376303:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000008:00100000:3.0:1787828602.376308:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828602.376310:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085641856:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1787828602.376322:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085641984:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787828602.376324:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498b40 x1874672085642496/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828602.376336:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828602.376339:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787828602.376341:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828602.376345:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2037 00000100:00080000:3.0:1787828602.376349:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828602.376354:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:1.0:1787828602.376357:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:3.0:1787828602.376359:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828602.376371:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a940 x1874672085642624/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828602.376377:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828602/real 1787828602] req@ffff9e35c649bc00 x1874672085641984/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828627 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828602.376390:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828602.376396:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:1.0:1787828602.376405:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649bc00 x1874672085641984/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828627 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828602.376416:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828602.376421:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828602.376425:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085641984:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828602.376464:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085642112:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787828602.376486:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828602.376499:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828602/real 1787828602] req@ffff9e35c649b840 x1874672085642112/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828627 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828602.376514:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b840 x1874672085642112/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828627 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828602.376518:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828602.376531:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2037 00000100:00080000:1.0:1787828602.376532:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828602.376537:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828602.376540:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828602.376545:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085642112:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828602.376551:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085642240:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828602.376574:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1787828602.376584:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828602.376590:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:1.0:1787828602.376593:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828602/real 1787828602] req@ffff9e35c6498f00 x1874672085642240/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828627 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828602.376605:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498780 x1874672085642752/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828602.376613:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498f00 x1874672085642240/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828627 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828602.376620:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828602.376628:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787828602.376632:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828602.376634:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828602.376638:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2037 00000100:00080000:1.0:1787828602.376644:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:3.0:1787828602.376644:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1787828602.376654:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085642240:192.168.203.141@tcp:8:kworker.0 00000100:00080000:3.0:1787828602.376654:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828602.376660:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:1.0:1787828602.376668:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085642368:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787828602.376673:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499a40 x1874672085642880/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787828602.376697:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828602.376710:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828602/real 1787828602] req@ffff9e35c6499680 x1874672085642368/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828627 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828602.376722:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499680 x1874672085642368/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828627 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828602.376732:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828602.376735:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828602.376739:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085642368:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828602.376745:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085642496:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787828602.376761:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828602.376770:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828602/real 1787828602] req@ffff9e35c6498b40 x1874672085642496/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828627 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828602.376783:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498b40 x1874672085642496/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828627 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828602.376792:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828602.376795:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828602.376836:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085642496:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828602.376844:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085642624:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787828602.376869:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828602.376883:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828602/real 1787828602] req@ffff9e35c649a940 x1874672085642624/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828627 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828602.376895:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a940 x1874672085642624/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828627 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828602.376904:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828602.376908:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828602.376911:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085642624:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828602.376930:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085642752:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828602.376947:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828602.376957:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828602/real 1787828602] req@ffff9e35c6498780 x1874672085642752/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828627 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828602.376970:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498780 x1874672085642752/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828627 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828602.376979:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828602.376982:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828602.377001:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085642752:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828602.377007:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085642880:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828602.377025:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828602.377035:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828602/real 1787828602] req@ffff9e35c6499a40 x1874672085642880/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828627 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828602.377048:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499a40 x1874672085642880/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828627 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828602.377057:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828602.377060:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828602.377064:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085642880:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787828602.503927:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828602.503927:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828602.503945:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787828602.504035:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828603.527890:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828603.527899:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787828603.527921:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787828603.527974:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828604.551887:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828604.551887:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787828604.552014:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787828604.552119:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828605.575897:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828605.575923:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828605.575932:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828605.575941:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828606.600006:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828606.600006:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787828606.600035:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828606.600058:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828607.495929:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828607.495943:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828607.495960:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828607.495973:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2042 00000100:00080000:0.0:1787828607.495987:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828607.495996:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787828607.496044:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718000 x1874672085643008/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828607.496079:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828607.496083:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828607.496085:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828607.496090:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2042 00000100:00080000:0.0:1787828607.496095:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828607.496100:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828607.496114:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828607.496124:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085643008:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1787828607.496133:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718780 x1874672085643136/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828607.496150:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828607.496154:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787828607.496156:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828607.496162:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2042 00000100:00080000:0.0:1787828607.496181:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828607.496187:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:1.0:1787828607.496193:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:0.0:1787828607.496200:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828607.496220:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da719a40 x1874672085643264/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828607.496233:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787828607.496234:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828607/real 1787828607] req@ffff9e35da718000 x1874672085643008/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828632 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828607.496245:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787828607.496252:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828607.496269:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2042 00000100:00100000:1.0:1787828607.496274:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718000 x1874672085643008/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828632 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828607.496277:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828607.496291:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828607.496297:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00080000:1.0:1787828607.496302:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828607.496308:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828607.496315:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a580 x1874672085643392/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828607.496317:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085643008:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1787828607.496343:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828607.496348:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787828607.496351:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828607.496359:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2042 00000100:00100000:1.0:1787828607.496362:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085643136:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787828607.496369:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828607.496378:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828607.496387:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000800:00000100:1.0:1787828607.496392:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828607.496407:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828607/real 1787828607] req@ffff9e35da718780 x1874672085643136/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828632 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828607.496412:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7183c0 x1874672085643520/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828607.496440:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718780 x1874672085643136/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828632 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828607.496448:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828607.496452:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828607.496455:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828607.496460:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:0.0:1787828607.496463:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787828607.496465:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085643136:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787828607.496479:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2042 00000100:00100000:1.0:1787828607.496481:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085643264:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787828607.496489:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828607.496497:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:1.0:1787828607.496503:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:0.0:1787828607.496504:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828607.496512:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828607/real 1787828607] req@ffff9e35da719a40 x1874672085643264/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828632 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828607.496522:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a1c0 x1874672085643648/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828607.496524:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672085643264/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828632 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828607.496586:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828607.496587:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828607.496597:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:0.0:1787828607.496598:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:1.0:1787828607.496605:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085643264:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787828607.496606:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828607.496627:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2042 00000100:00100000:1.0:1787828607.496628:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085643392:192.168.203.141@tcp:8:kworker.0 00000100:00080000:0.0:1787828607.496639:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828607.496648:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:1.0:1787828607.496654:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:0.0:1787828607.496655:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828607.496663:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828607/real 1787828607] req@ffff9e35da71a580 x1874672085643392/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828632 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828607.496674:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a940 x1874672085643776/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828607.496675:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672085643392/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828632 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828607.496703:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828607.496706:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828607.496708:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085643392:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828607.496726:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085643520:192.168.203.141@tcp:8:kworker.0 00000100:00080000:0.0:1787828607.496749:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828607.496755:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000100:1.0:1787828607.496756:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787828607.496766:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787828607.496772:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828607/real 1787828607] req@ffff9e35da7183c0 x1874672085643520/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828632 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828607.496776:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2042 00000100:00080000:0.0:1787828607.496790:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1787828607.496817:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672085643520/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828632 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828607.496825:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828607.496829:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828607.496831:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085643520:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828607.496837:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085643648:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787828607.496847:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:1.0:1787828607.496853:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:0.0:1787828607.496857:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:1.0:1787828607.496864:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828607/real 1787828607] req@ffff9e35da71a1c0 x1874672085643648/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828632 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828607.496879:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718f00 x1874672085643904/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828607.496883:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a1c0 x1874672085643648/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828632 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828607.496940:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828607.496943:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828607.496944:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828607.496965:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085643648:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787828607.496970:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787828607.496974:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828607.496981:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2042 00000100:00100000:1.0:1787828607.496985:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085643776:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787828607.496989:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828607.496997:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828607.497003:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000800:00000100:1.0:1787828607.497010:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828607.497020:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718b40 x1874672085644032/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828607.497023:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828607/real 1787828607] req@ffff9e35da71a940 x1874672085643776/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828632 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828607.497092:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672085643776/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828632 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828607.497100:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828607.497103:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828607.497106:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085643776:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828607.497110:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085643904:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828607.497126:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828607.497133:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828607/real 1787828607] req@ffff9e35da718f00 x1874672085643904/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828632 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828607.497140:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672085643904/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828632 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828607.497150:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828607.497152:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828607.497154:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085643904:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828607.497163:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085644032:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828607.497172:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828607.497178:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828607/real 1787828607] req@ffff9e35da718b40 x1874672085644032/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828632 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828607.497185:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718b40 x1874672085644032/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828632 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828607.497190:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828607.497192:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828607.497194:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085644032:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828607.623879:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828607.623895:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787828607.623922:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828607.623924:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828608.647938:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787828608.647958:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787828608.647992:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828608.648040:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828609.671905:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787828609.671905:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828609.671905:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828609.671975:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828610.695905:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828610.695906:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828610.695947:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828610.695962:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828611.719915:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828611.719915:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828611.719969:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828611.719972:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828612.615904:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828612.615929:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828612.615935:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828612.615949:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2047 00000100:00080000:0.0:1787828612.615962:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828612.615970:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787828612.616016:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7192c0 x1874672085644160/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828612.616051:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828612.616055:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828612.616057:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828612.616061:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2047 00000100:00080000:0.0:1787828612.616065:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828612.616069:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828612.616082:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828612.616098:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718b40 x1874672085644288/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828612.616110:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085644160:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1787828612.616114:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828612.616119:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787828612.616122:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828612.616126:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2047 00000100:00080000:0.0:1787828612.616130:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828612.616134:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828612.616140:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828612.616153:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718f00 x1874672085644416/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828612.616165:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828612.616168:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787828612.616171:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828612.616176:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2047 00000100:00080000:0.0:1787828612.616180:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828612.616185:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:1.0:1787828612.616189:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:0.0:1787828612.616194:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:0.0:1787828612.616210:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a940 x1874672085644544/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828612.616220:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828612.616224:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787828612.616227:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828612.616232:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2047 00000100:00100000:1.0:1787828612.616235:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828612/real 1787828612] req@ffff9e35da7192c0 x1874672085644160/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828637 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828612.616238:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828612.616247:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828612.616254:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:1.0:1787828612.616258:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7192c0 x1874672085644160/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828637 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828612.616269:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a1c0 x1874672085644672/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828612.616274:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828612.616285:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:0.0:1787828612.616289:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787828612.616293:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085644160:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1787828612.616294:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828612.616298:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828612.616306:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2047 00000100:00080000:0.0:1787828612.616311:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828612.616315:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828612.616320:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828612.616332:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085644288:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787828612.616334:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7183c0 x1874672085644800/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828612.616348:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828612.616352:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787828612.616355:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1787828612.616360:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787828612.616364:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2047 00000100:00080000:0.0:1787828612.616371:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828612.616378:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787828612.616378:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828612/real 1787828612] req@ffff9e35da718b40 x1874672085644288/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828637 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1787828612.616388:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828612.616393:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718b40 x1874672085644288/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828637 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828612.616401:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1787828612.616405:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a580 x1874672085644928/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828612.616407:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828612.616414:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085644288:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828612.616425:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085644416:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787828612.616428:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828612.616434:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787828612.616436:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828612.616442:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2047 00000800:00000100:1.0:1787828612.616446:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787828612.616449:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828612.616457:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787828612.616458:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828612/real 1787828612] req@ffff9e35da718f00 x1874672085644416/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828637 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1787828612.616466:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:1.0:1787828612.616471:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672085644416/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828637 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828612.616478:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828612.616484:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828612.616486:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da719a40 x1874672085645056/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828612.616488:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085644416:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828612.616500:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085644544:192.168.203.141@tcp:8:kworker.0 00000100:00080000:0.0:1787828612.616517:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828612.616523:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000800:00000100:1.0:1787828612.616523:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787828612.616527:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828612.616534:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2047 00000100:00100000:1.0:1787828612.616534:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828612/real 1787828612] req@ffff9e35da71a940 x1874672085644544/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828637 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828612.616541:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828612.616549:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787828612.616551:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672085644544/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828637 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1787828612.616557:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00080000:1.0:1787828612.616561:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828612.616565:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828612.616568:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085644544:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828612.616576:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085644672:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787828612.616577:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718780 x1874672085645184/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787828612.616592:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828612.616601:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828612/real 1787828612] req@ffff9e35da71a1c0 x1874672085644672/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828637 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828612.616615:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a1c0 x1874672085644672/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828637 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828612.616622:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828612.616624:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828612.616626:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085644672:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828612.616640:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085644800:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787828612.616652:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828612.616659:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828612/real 1787828612] req@ffff9e35da7183c0 x1874672085644800/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828637 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828612.616667:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672085644800/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828637 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828612.616672:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828612.616674:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828612.616676:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085644800:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828612.616679:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085644928:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787828612.616689:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828612.616695:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828612/real 1787828612] req@ffff9e35da71a580 x1874672085644928/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828637 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828612.616703:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672085644928/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828637 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828612.616708:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828612.616710:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828612.616711:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085644928:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828612.616716:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085645056:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828612.616733:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828612.616744:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828612/real 1787828612] req@ffff9e35da719a40 x1874672085645056/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828637 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828612.616757:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672085645056/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828637 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828612.616772:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828612.616775:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828612.616777:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085645056:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828612.616781:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085645184:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828612.616820:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828612.616829:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828612/real 1787828612] req@ffff9e35da718780 x1874672085645184/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828637 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828612.616837:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718780 x1874672085645184/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828637 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828612.616843:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828612.616846:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828612.616848:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085645184:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828612.743930:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828612.743930:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828612.743970:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828612.744049:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787828613.767890:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828613.767890:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828613.767899:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828613.767936:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828614.791893:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828614.791893:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787828614.791912:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828614.791943:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828615.815904:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828615.815905:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828615.815932:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828615.815937:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828616.839903:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828616.839908:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828616.839912:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828616.839958:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828617.735920:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828617.735935:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828617.735939:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828617.735949:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2053 00000100:00080000:2.0:1787828617.735960:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828617.735968:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787828617.736003:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498d680 x1874672085645312/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828617.736031:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828617.736035:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828617.736037:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828617.736042:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2053 00000100:00080000:2.0:1787828617.736046:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828617.736050:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828617.736061:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828617.736075:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498fc00 x1874672085645440/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828617.736081:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085645312:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1787828617.736094:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828617.736099:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787828617.736102:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828617.736107:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2053 00000100:00080000:2.0:1787828617.736111:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828617.736115:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828617.736121:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828617.736133:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498de00 x1874672085645568/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828617.736143:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1787828617.736147:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1787828617.736149:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787828617.736153:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828617.736158:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2053 00000100:00080000:2.0:1787828617.736162:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828617.736167:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828617.736172:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:2.0:1787828617.736184:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498ed00 x1874672085645696/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828617.736191:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828617/real 1787828617] req@ffff9e35c498d680 x1874672085645312/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828642 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828617.736201:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828617.736207:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787828617.736210:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828617.736215:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2053 00000100:00080000:2.0:1787828617.736218:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828617.736221:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828617.736225:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:1.0:1787828617.736230:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498d680 x1874672085645312/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828642 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828617.736236:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c3c0 x1874672085645824/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828617.736256:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828617.736256:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828617.736262:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828617.736265:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828617.736266:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:2.0:1787828617.736271:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2053 00000100:00080000:2.0:1787828617.736275:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1787828617.736279:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085645312:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1787828617.736279:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828617.736281:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828617.736293:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e940 x1874672085645952/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828617.736299:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828617.736302:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787828617.736303:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828617.736306:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2053 00000100:00080000:2.0:1787828617.736308:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1787828617.736311:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085645440:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1787828617.736313:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828617.736320:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828617.736330:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c780 x1874672085646080/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828617.736337:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828617.736340:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787828617.736342:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1787828617.736347:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1787828617.736350:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2053 00000100:00080000:2.0:1787828617.736355:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828617.736359:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828617.736363:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:1.0:1787828617.736369:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828617/real 1787828617] req@ffff9e35c498fc00 x1874672085645440/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828642 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828617.736374:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498d680 x1874672085646208/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828617.736399:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828617.736401:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787828617.736405:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787828617.736406:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498fc00 x1874672085645440/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828642 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828617.736410:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2053 00000100:00080000:2.0:1787828617.736414:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828617.736420:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828617.736428:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00080000:1.0:1787828617.736430:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828617.736437:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828617.736438:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cf00 x1874672085646336/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828617.736445:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085645440:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828617.736456:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085645568:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787828617.736477:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828617.736488:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828617/real 1787828617] req@ffff9e35c498de00 x1874672085645568/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828642 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828617.736501:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498de00 x1874672085645568/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828642 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828617.736518:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828617.736521:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828617.736525:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085645568:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828617.736531:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085645696:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828617.736549:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828617.736565:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828617/real 1787828617] req@ffff9e35c498ed00 x1874672085645696/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828642 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828617.736577:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498ed00 x1874672085645696/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828642 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828617.736586:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828617.736589:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828617.736593:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085645696:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828617.736599:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085645824:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828617.736616:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828617.736625:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828617/real 1787828617] req@ffff9e35c498c3c0 x1874672085645824/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828642 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828617.736637:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c3c0 x1874672085645824/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828642 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828617.736645:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828617.736649:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828617.736652:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085645824:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828617.736657:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085645952:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787828617.736683:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828617.736697:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828617/real 1787828617] req@ffff9e35c498e940 x1874672085645952/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828642 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828617.736710:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e940 x1874672085645952/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828642 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828617.736720:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828617.736723:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828617.736726:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085645952:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828617.736747:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085646080:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787828617.736765:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828617.736775:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828617/real 1787828617] req@ffff9e35c498c780 x1874672085646080/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828642 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828617.736788:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c780 x1874672085646080/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828642 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828617.736822:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828617.736827:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828617.736831:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085646080:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828617.736837:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085646208:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828617.736856:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828617.736867:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828617/real 1787828617] req@ffff9e35c498d680 x1874672085646208/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828642 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828617.736892:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498d680 x1874672085646208/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828642 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828617.736908:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828617.736911:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828617.736915:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085646208:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828617.736920:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085646336:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828617.736945:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828617.736955:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828617/real 1787828617] req@ffff9e35c498cf00 x1874672085646336/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828642 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828617.736967:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498cf00 x1874672085646336/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828642 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828617.736975:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828617.736979:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828617.736982:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085646336:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787828617.863926:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828617.863982:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828617.864033:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787828617.864071:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787828618.887924:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828618.887931:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828618.887999:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828618.888054:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828619.911858:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787828619.911859:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828619.911886:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828619.911929:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828620.935884:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828620.935886:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787828620.935911:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828620.935938:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828621.959841:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828621.959884:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828621.959903:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828621.959912:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828622.855914:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828622.855925:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787828622.855930:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828622.855941:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2058 00000100:00080000:3.0:1787828622.855952:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828622.855960:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787828622.856005:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499a40 x1874672085646464/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828622.856036:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828622.856039:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828622.856041:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828622.856044:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2058 00000100:00080000:3.0:1787828622.856047:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828622.856049:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828622.856059:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828622.856067:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498780 x1874672085646592/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828622.856079:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828622.856081:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787828622.856083:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828622.856086:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2058 00000100:00080000:3.0:1787828622.856088:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828622.856091:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828622.856094:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828622.856101:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a940 x1874672085646720/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828622.856108:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828622.856111:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787828622.856113:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828622.856117:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2058 00000100:00080000:3.0:1787828622.856119:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828622.856122:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828622.856125:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:1.0:1787828622.856130:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085646464:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1787828622.856135:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498b40 x1874672085646848/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828622.856143:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828622.856145:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787828622.856146:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828622.856149:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2058 00000100:00080000:3.0:1787828622.856151:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828622.856153:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828622.856156:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:3.0:1787828622.856165:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499680 x1874672085646976/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828622.856171:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828622.856174:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828622.856175:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828622.856179:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2058 00000100:00080000:3.0:1787828622.856182:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828622.856184:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828622.856187:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828622.856194:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498f00 x1874672085647104/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828622.856201:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828622.856203:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787828622.856207:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1787828622.856208:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1787828622.856212:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2058 00000100:00080000:3.0:1787828622.856216:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828622.856218:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828622.856223:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828622.856230:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b840 x1874672085647232/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828622.856239:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828622.856243:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787828622.856245:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828622.856251:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2058 00000100:00080000:3.0:1787828622.856256:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1787828622.856260:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828622/real 1787828622] req@ffff9e35c6499a40 x1874672085646464/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828647 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828622.856264:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828622.856275:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:1.0:1787828622.856284:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499a40 x1874672085646464/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828647 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828622.856292:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649bc00 x1874672085647360/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828622.856306:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828622.856312:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:3.0:1787828622.856312:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828622.856321:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:1.0:1787828622.856322:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085646464:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1787828622.856327:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787828622.856332:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085646592:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1787828622.856337:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2058 00000100:00080000:3.0:1787828622.856343:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828622.856346:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828622.856351:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000800:00000100:1.0:1787828622.856355:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828622.856366:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a580 x1874672085647488/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828622.856369:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828622/real 1787828622] req@ffff9e35c6498780 x1874672085646592/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828647 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828622.856397:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498780 x1874672085646592/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828647 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828622.856405:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828622.856407:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828622.856409:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085646592:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828622.856414:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085646720:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787828622.856427:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828622.856433:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828622/real 1787828622] req@ffff9e35c649a940 x1874672085646720/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828647 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828622.856441:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a940 x1874672085646720/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828647 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828622.856446:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828622.856448:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828622.856450:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085646720:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828622.856477:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085646848:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828622.856489:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828622.856495:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828622/real 1787828622] req@ffff9e35c6498b40 x1874672085646848/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828647 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828622.856503:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498b40 x1874672085646848/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828647 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828622.856509:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828622.856510:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828622.856512:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085646848:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828622.856516:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085646976:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828622.856525:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828622.856531:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828622/real 1787828622] req@ffff9e35c6499680 x1874672085646976/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828647 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828622.856538:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499680 x1874672085646976/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828647 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828622.856544:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828622.856545:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828622.856547:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085646976:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828622.856551:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085647104:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787828622.856560:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828622.856566:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828622/real 1787828622] req@ffff9e35c6498f00 x1874672085647104/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828647 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828622.856574:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498f00 x1874672085647104/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828647 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828622.856579:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828622.856581:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828622.856583:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085647104:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828622.856586:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085647232:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787828622.856596:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828622.856602:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828622/real 1787828622] req@ffff9e35c649b840 x1874672085647232/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828647 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828622.856610:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b840 x1874672085647232/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828647 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828622.856615:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828622.856617:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828622.856618:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085647232:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828622.856621:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085647360:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828622.856631:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828622.856636:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828622/real 1787828622] req@ffff9e35c649bc00 x1874672085647360/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828647 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828622.856643:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649bc00 x1874672085647360/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828647 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828622.856653:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828622.856655:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828622.856657:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085647360:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828622.856660:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085647488:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828622.856669:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828622.856675:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828622/real 1787828622] req@ffff9e35c649a580 x1874672085647488/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828647 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828622.856682:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a580 x1874672085647488/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828647 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828622.856688:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828622.856689:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828622.856691:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085647488:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787828622.983934:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787828622.983934:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828622.983939:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828622.984008:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828624.007880:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828624.007886:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787828624.007887:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828624.007924:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828625.031873:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828625.031879:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828625.031960:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828625.031981:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828626.055897:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828626.055906:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828626.055958:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828626.055967:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828627.079891:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828627.079896:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787828627.079932:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787828627.079933:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828627.975909:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828627.975917:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828627.975920:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828627.975928:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2063 00000100:00080000:1.0:1787828627.975939:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828627.975945:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787828627.975976:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6d00 x1874672085647616/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828627.975999:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828627.976002:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828627.976003:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828627.976006:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2063 00000100:00080000:1.0:1787828627.976008:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828627.976012:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828627.976020:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828627.976029:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5e00 x1874672085647744/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787828627.976035:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085647616:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787828627.976041:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828627.976045:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787828627.976046:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828627.976050:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2063 00000100:00080000:1.0:1787828627.976052:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828627.976054:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828627.976058:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828627.976067:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7480 x1874672085647872/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828627.976073:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828627.976076:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828627.976078:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828627.976080:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2063 00000100:00080000:1.0:1787828627.976084:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828627.976086:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828627.976089:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:1.0:1787828627.976099:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4000 x1874672085648000/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787828627.976100:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787828627.976110:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828627.976113:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828627.976114:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828627.976118:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2063 00000100:00080000:1.0:1787828627.976122:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828627.976124:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828627.976127:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:1.0:1787828627.976134:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f43c0 x1874672085648128/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828627.976141:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1787828627.976144:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828627/real 1787828627] req@ffff9e35da6f6d00 x1874672085647616/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828652 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828627.976145:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828627.976148:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828627.976156:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2063 00000100:00080000:1.0:1787828627.976161:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828627.976164:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828627.976167:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828627.976169:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6d00 x1874672085647616/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828652 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828627.976179:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4b40 x1874672085648256/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828627.976180:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828627.976189:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:1.0:1787828627.976195:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1787828627.976197:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085647616:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787828627.976199:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787828627.976202:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828627.976207:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2063 00000100:00080000:1.0:1787828627.976209:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828627.976211:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828627.976214:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828627.976221:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4f00 x1874672085648384/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787828627.976225:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085647744:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787828627.976233:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828627.976236:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828627.976237:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828627.976243:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2063 00000100:00080000:1.0:1787828627.976245:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828627.976247:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828627.976251:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000800:00000100:3.0:1787828627.976262:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828627.976263:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7840 x1874672085648512/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828627.976272:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828627.976274:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828627.976277:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1787828627.976278:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828627/real 1787828627] req@ffff9e35da6f5e00 x1874672085647744/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828652 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828627.976283:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2063 00000100:00080000:1.0:1787828627.976289:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828627.976295:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828627.976301:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:3.0:1787828627.976311:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5e00 x1874672085647744/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828652 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828627.976315:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4780 x1874672085648640/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828627.976334:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828627.976339:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828627.976342:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085647744:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787828627.976348:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085647872:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787828627.976364:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828627.976371:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828627/real 1787828627] req@ffff9e35da6f7480 x1874672085647872/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828652 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828627.976379:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7480 x1874672085647872/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828652 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828627.976384:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828627.976388:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828627.976391:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085647872:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787828627.976397:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085648000:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787828627.976416:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828627.976427:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828627/real 1787828627] req@ffff9e35da6f4000 x1874672085648000/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828652 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828627.976440:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4000 x1874672085648000/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828652 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828627.976453:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828627.976455:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828627.976458:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085648000:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787828627.976462:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085648128:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787828627.976476:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828627.976484:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828627/real 1787828627] req@ffff9e35da6f43c0 x1874672085648128/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828652 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828627.976491:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f43c0 x1874672085648128/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828652 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828627.976497:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828627.976499:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828627.976501:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085648128:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787828627.976504:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085648256:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787828627.976514:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828627.976527:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828627/real 1787828627] req@ffff9e35da6f4b40 x1874672085648256/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828652 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828627.976535:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4b40 x1874672085648256/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828652 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828627.976540:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828627.976542:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828627.976545:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085648256:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787828627.976562:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085648384:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787828627.976574:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828627.976581:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828627/real 1787828627] req@ffff9e35da6f4f00 x1874672085648384/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828652 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828627.976588:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4f00 x1874672085648384/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828652 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828627.976593:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828627.976595:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828627.976597:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085648384:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787828627.976606:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085648512:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787828627.976617:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828627.976623:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828627/real 1787828627] req@ffff9e35da6f7840 x1874672085648512/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828652 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828627.976630:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7840 x1874672085648512/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828652 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828627.976635:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828627.976637:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828627.976639:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085648512:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787828627.976642:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085648640:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787828627.976652:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828627.976658:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828627/real 1787828627] req@ffff9e35da6f4780 x1874672085648640/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828652 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828627.976665:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4780 x1874672085648640/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828652 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828627.976670:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828627.976672:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828627.976674:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085648640:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828628.103876:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828628.103887:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828628.103932:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828628.103963:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828629.127903:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787828629.127933:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828629.127952:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828629.127957:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787828630.151905:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828630.151905:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787828630.151934:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828630.152079:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828631.175888:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787828631.175894:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828631.175911:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828631.175921:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828632.199912:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828632.199942:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787828632.199970:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828632.200035:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828633.095992:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828633.096005:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828633.096010:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828633.096023:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2068 00000100:00080000:1.0:1787828633.096049:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828633.096057:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787828633.096109:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f61c0 x1874672085648768/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828633.096162:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828633.096168:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828633.096179:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828633.096186:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2068 00000100:00080000:1.0:1787828633.096193:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828633.096212:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828633.096228:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828633.096239:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085648768:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1787828633.096252:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5a40 x1874672085648896/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828633.096271:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828633.096276:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787828633.096280:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828633.096285:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2068 00000100:00080000:1.0:1787828633.096292:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828633.096298:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828633.096306:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1787828633.096328:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828633.096329:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6580 x1874672085649024/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828633.096347:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828633.096352:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828633.096356:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828633.096362:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2068 00000100:00080000:1.0:1787828633.096368:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828633.096374:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828633.096382:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:3.0:1787828633.096394:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828633/real 1787828633] req@ffff9e35da6f61c0 x1874672085648768/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828658 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828633.096401:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6940 x1874672085649152/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828633.096434:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828633.096442:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:3.0:1787828633.096443:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f61c0 x1874672085648768/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828658 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828633.096447:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828633.096463:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2068 00000100:00080000:3.0:1787828633.096466:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828633.096470:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828633.096474:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:1.0:1787828633.096482:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787828633.096483:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085648768:192.168.203.141@tcp:250:kworker.0 00000008:00100000:1.0:1787828633.096496:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:1.0:1787828633.096518:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f70c0 x1874672085649280/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787828633.096522:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085648896:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787828633.096545:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828633.096551:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828633.096554:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828633.096562:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2068 00000100:00080000:1.0:1787828633.096568:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828633.096574:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:3.0:1787828633.096578:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:1.0:1787828633.096586:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828633.096598:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828633/real 1787828633] req@ffff9e35da6f5a40 x1874672085648896/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828658 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828633.096609:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f61c0 x1874672085649408/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787828633.096619:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5a40 x1874672085648896/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828658 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828633.096629:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828633.096636:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787828633.096637:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828633.096641:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828633.096643:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:1.0:1787828633.096656:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2068 00000100:00100000:3.0:1787828633.096656:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085648896:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787828633.096668:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1787828633.096671:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085649024:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787828633.096679:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828633.096689:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1787828633.096697:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828633.096727:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4780 x1874672085649536/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787828633.096732:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828633/real 1787828633] req@ffff9e35da6f6580 x1874672085649024/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828658 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828633.096768:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828633.096850:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828633.096853:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1787828633.096860:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6580 x1874672085649024/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828658 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828633.096864:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2068 00000100:00080000:1.0:1787828633.096874:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828633.096880:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828633.096885:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:3.0:1787828633.096889:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828633.096898:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085649024:192.168.203.141@tcp:38:kworker.0 00000008:00100000:1.0:1787828633.096899:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:3.0:1787828633.096907:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085649152:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828633.096921:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7840 x1874672085649664/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828633.096954:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:3.0:1787828633.096971:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787828633.096975:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828633.096980:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828633.096989:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2068 00000100:00080000:1.0:1787828633.096996:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1787828633.096999:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828633/real 1787828633] req@ffff9e35da6f6940 x1874672085649152/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828658 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828633.097007:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828633.097034:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:3.0:1787828633.097038:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6940 x1874672085649152/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828658 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828633.097058:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1787828633.097060:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4f00 x1874672085649792/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828633.097066:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828633.097077:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085649152:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787828633.097112:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085649280:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787828633.097153:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828633.097163:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828633/real 1787828633] req@ffff9e35da6f70c0 x1874672085649280/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828658 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828633.097173:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f70c0 x1874672085649280/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828658 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828633.097181:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828633.097183:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828633.097186:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085649280:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787828633.097199:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085649408:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787828633.097212:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828633.097219:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828633/real 1787828633] req@ffff9e35da6f61c0 x1874672085649408/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828658 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828633.097229:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f61c0 x1874672085649408/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828658 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828633.097236:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828633.097239:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828633.097242:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085649408:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787828633.097248:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085649536:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787828633.097260:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828633.097267:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828633/real 1787828633] req@ffff9e35da6f4780 x1874672085649536/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828658 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828633.097277:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4780 x1874672085649536/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828658 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828633.097284:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828633.097286:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828633.097288:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085649536:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787828633.097293:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085649664:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787828633.097305:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828633.097313:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828633/real 1787828633] req@ffff9e35da6f7840 x1874672085649664/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828658 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828633.097323:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7840 x1874672085649664/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828658 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828633.097330:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828633.097332:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828633.097334:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085649664:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787828633.097339:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085649792:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787828633.097358:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828633.097372:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828633/real 1787828633] req@ffff9e35da6f4f00 x1874672085649792/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828658 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828633.097388:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4f00 x1874672085649792/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828658 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828633.097401:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828633.097405:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828633.097410:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085649792:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787828633.223935:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828633.223935:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828633.223946:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828633.223977:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828634.247929:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828634.247929:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828634.247940:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787828634.247980:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828635.271925:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787828635.271928:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787828635.271982:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828635.272004:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828636.295897:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787828636.295899:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787828636.295903:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828636.295972:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828637.319974:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828637.320020:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787828637.320177:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787828637.320325:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828638.215960:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828638.215972:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828638.215977:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828638.215989:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2073 00000100:00080000:0.0:1787828638.216001:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828638.216010:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787828638.216073:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718000 x1874672085649920/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828638.216107:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828638.216112:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828638.216115:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828638.216123:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2073 00000100:00080000:0.0:1787828638.216130:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828638.216137:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787828638.216150:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085649920:192.168.203.141@tcp:250:kworker.0 00000008:00100000:0.0:1787828638.216151:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828638.216189:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718780 x1874672085650048/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828638.216207:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828638.216212:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787828638.216216:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1787828638.216235:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787828638.216251:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2073 00000100:00080000:0.0:1787828638.216259:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828638.216266:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828638.216273:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828638.216281:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828638/real 1787828638] req@ffff9e35da718000 x1874672085649920/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828663 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828638.216292:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da719a40 x1874672085650176/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828638.216302:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718000 x1874672085649920/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828663 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828638.216312:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828638.216317:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828638.216319:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828638.216323:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:0.0:1787828638.216325:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787828638.216329:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085649920:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1787828638.216336:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2073 00000100:00080000:0.0:1787828638.216343:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828638.216348:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787828638.216355:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085650048:192.168.203.141@tcp:38:kworker.0 00000008:00100000:0.0:1787828638.216357:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000800:00000100:1.0:1787828638.216382:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828638.216389:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a580 x1874672085650304/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828638.216396:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828638/real 1787828638] req@ffff9e35da718780 x1874672085650048/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828663 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828638.216422:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787828638.216426:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718780 x1874672085650048/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828663 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828638.216429:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787828638.216434:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828638.216446:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2073 00000100:00080000:1.0:1787828638.216451:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828638.216453:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828638.216461:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:0.0:1787828638.216463:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787828638.216472:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085650048:192.168.203.141@tcp:38:kworker.0 00000008:00100000:0.0:1787828638.216473:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:1.0:1787828638.216487:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085650176:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787828638.216496:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7183c0 x1874672085650432/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828638.216511:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1787828638.216513:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787828638.216520:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828638.216524:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787828638.216530:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828638/real 1787828638] req@ffff9e35da719a40 x1874672085650176/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828663 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828638.216532:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2073 00000100:00080000:0.0:1787828638.216543:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828638.216554:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787828638.216562:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672085650176/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828663 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1787828638.216563:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00080000:1.0:1787828638.216580:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828638.216584:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828638.216590:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a1c0 x1874672085650560/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828638.216591:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085650176:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787828638.216610:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828638.216615:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787828638.216618:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787828638.216625:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085650304:192.168.203.141@tcp:8:kworker.0 00000100:00080000:0.0:1787828638.216628:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2073 00000100:00080000:0.0:1787828638.216638:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828638.216661:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828638.216672:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1787828638.216679:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828638.216700:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a940 x1874672085650688/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828638.216700:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828638/real 1787828638] req@ffff9e35da71a580 x1874672085650304/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828663 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828638.216765:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672085650304/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828663 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828638.216767:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828638.216780:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787828638.216787:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828638.216789:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828638.216836:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:0.0:1787828638.216841:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2073 00000100:00100000:1.0:1787828638.216845:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085650304:192.168.203.141@tcp:8:kworker.0 00000100:00080000:0.0:1787828638.216851:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1787828638.216860:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085650432:192.168.203.141@tcp:8:kworker.0 00000100:00080000:0.0:1787828638.216862:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828638.216877:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:0.0:1787828638.216904:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718f00 x1874672085650816/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787828638.216909:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787828638.216927:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787828638.216931:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828638/real 1787828638] req@ffff9e35da7183c0 x1874672085650432/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828663 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828638.216936:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787828638.216943:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787828638.216954:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672085650432/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828663 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828638.216955:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2073 00000100:00080000:0.0:1787828638.216967:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828638.216978:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:1.0:1787828638.216982:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000008:00100000:0.0:1787828638.216988:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00080000:1.0:1787828638.216989:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828638.216996:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085650432:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828638.217004:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085650560:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787828638.217015:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718b40 x1874672085650944/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787828638.217045:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828638.217060:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828638/real 1787828638] req@ffff9e35da71a1c0 x1874672085650560/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828663 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828638.217072:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a1c0 x1874672085650560/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828663 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828638.217080:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828638.217083:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828638.217087:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085650560:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828638.217102:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085650688:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787828638.217116:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828638.217124:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828638/real 1787828638] req@ffff9e35da71a940 x1874672085650688/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828663 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828638.217133:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672085650688/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828663 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828638.217139:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828638.217141:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828638.217144:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085650688:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828638.217148:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085650816:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828638.217160:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828638.217167:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828638/real 1787828638] req@ffff9e35da718f00 x1874672085650816/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828663 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828638.217176:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672085650816/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828663 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828638.217182:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828638.217184:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828638.217186:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085650816:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828638.217190:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085650944:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828638.217223:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828638.217235:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828638/real 1787828638] req@ffff9e35da718b40 x1874672085650944/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828663 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828638.217250:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718b40 x1874672085650944/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828663 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828638.217260:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828638.217264:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828638.217285:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085650944:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828638.343909:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787828638.343918:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787828638.343941:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:0.0:1787828638.344029:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828639.367891:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828639.367954:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787828639.367965:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828639.367997:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828640.391925:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787828640.391956:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828640.391968:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828640.391969:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828641.415956:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828641.415956:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787828641.415992:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828641.416061:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:3.0:1787828642.439904:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787828642.439905:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828642.439951:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787828642.439995:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828643.335899:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828643.335910:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828643.335915:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828643.335925:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2078 00000100:00080000:2.0:1787828643.335937:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828643.335944:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787828643.335988:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e580 x1874672085651072/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828643.336018:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828643.336021:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828643.336023:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828643.336026:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2078 00000100:00080000:2.0:1787828643.336029:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828643.336032:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828643.336043:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828643.336057:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f840 x1874672085651200/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828643.336065:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085651072:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1787828643.336076:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828643.336079:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787828643.336081:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828643.336085:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2078 00000100:00080000:2.0:1787828643.336089:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828643.336093:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828643.336097:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828643.336106:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c000 x1874672085651328/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828643.336113:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828643.336117:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787828643.336118:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828643.336122:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2078 00000100:00080000:2.0:1787828643.336125:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828643.336129:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828643.336134:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000800:00000100:1.0:1787828643.336140:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828643.336145:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cb40 x1874672085651456/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828643.336155:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828643.336159:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787828643.336160:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828643.336164:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2078 00000100:00080000:2.0:1787828643.336167:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828643.336170:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828643.336175:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:1.0:1787828643.336186:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828643/real 1787828643] req@ffff9e35c498e580 x1874672085651072/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828668 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828643.336187:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e1c0 x1874672085651584/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828643.336239:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828643.336252:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828643.336255:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828643.336263:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2078 00000100:00100000:1.0:1787828643.336270:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e580 x1874672085651072/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828668 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828643.336270:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828643.336281:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828643.336290:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00080000:1.0:1787828643.336304:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:2.0:1787828643.336310:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f480 x1874672085651712/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828643.336312:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828643.336329:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085651072:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1787828643.336342:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828643.336346:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787828643.336348:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828643.336353:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2078 00000100:00080000:2.0:1787828643.336355:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828643.336358:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828643.336363:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828643.336371:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f0c0 x1874672085651840/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828643.336378:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085651200:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1787828643.336382:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828643.336389:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787828643.336391:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828643.336399:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2078 00000100:00080000:2.0:1787828643.336403:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828643.336407:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828643.336411:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:2.0:1787828643.336422:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e580 x1874672085651968/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787828643.336423:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1787828643.336438:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828643.336442:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787828643.336444:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828643.336451:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2078 00000100:00100000:1.0:1787828643.336452:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828643/real 1787828643] req@ffff9e35c498f840 x1874672085651200/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828668 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828643.336461:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828643.336472:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787828643.336475:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f840 x1874672085651200/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828668 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1787828643.336486:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00080000:1.0:1787828643.336488:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828643.336495:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828643.336499:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085651200:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828643.336506:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085651328:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828643.336513:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cf00 x1874672085652096/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787828643.336532:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828643.336543:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828643/real 1787828643] req@ffff9e35c498c000 x1874672085651328/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828668 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828643.336564:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c000 x1874672085651328/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828668 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828643.336570:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828643.336574:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828643.336577:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085651328:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828643.336584:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085651456:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828643.336605:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828643.336617:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828643/real 1787828643] req@ffff9e35c498cb40 x1874672085651456/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828668 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828643.336630:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498cb40 x1874672085651456/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828668 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828643.336640:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828643.336643:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828643.336645:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085651456:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828643.336650:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085651584:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828643.336666:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828643.336673:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828643/real 1787828643] req@ffff9e35c498e1c0 x1874672085651584/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828668 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828643.336686:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e1c0 x1874672085651584/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828668 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828643.336692:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828643.336694:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828643.336696:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085651584:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828643.336719:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085651712:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787828643.336738:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828643.336746:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828643/real 1787828643] req@ffff9e35c498f480 x1874672085651712/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828668 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828643.336754:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f480 x1874672085651712/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828668 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828643.336759:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828643.336762:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828643.336764:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085651712:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828643.336782:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085651840:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787828643.336795:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828643.336821:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828643/real 1787828643] req@ffff9e35c498f0c0 x1874672085651840/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828668 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828643.336829:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f0c0 x1874672085651840/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828668 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828643.336838:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828643.336841:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828643.336843:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085651840:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828643.336847:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085651968:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828643.336859:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828643.336865:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828643/real 1787828643] req@ffff9e35c498e580 x1874672085651968/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828668 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828643.336872:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e580 x1874672085651968/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828668 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828643.336877:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828643.336879:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828643.336881:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085651968:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828643.336885:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085652096:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828643.336894:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828643.336900:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828643/real 1787828643] req@ffff9e35c498cf00 x1874672085652096/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828668 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828643.336907:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498cf00 x1874672085652096/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828668 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828643.336912:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828643.336914:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828643.336916:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085652096:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787828643.463939:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828643.463939:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787828643.463997:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828643.463998:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787828644.487910:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787828644.487910:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787828644.487964:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828644.488036:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828645.511915:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787828645.511915:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787828645.511959:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828645.511974:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828646.535967:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828646.535973:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787828646.536037:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828646.536037:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787828647.559875:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828647.559883:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787828647.559898:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787828647.559930:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828648.455925:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828648.455933:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828648.455936:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828648.455944:0:27606:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2083 00000100:00080000:1.0:1787828648.455954:0:27606:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828648.455960:0:27606:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787828648.455992:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4b40 x1874672085652224/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828648.456018:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828648.456021:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828648.456022:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828648.456025:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2083 00000100:00080000:1.0:1787828648.456027:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828648.456030:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828648.456038:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828648.456047:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f43c0 x1874672085652352/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828648.456055:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828648.456060:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:3.0:1787828648.456061:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085652224:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787828648.456062:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828648.456070:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2083 00000100:00080000:1.0:1787828648.456073:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828648.456075:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828648.456078:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828648.456088:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4000 x1874672085652480/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828648.456094:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828648.456097:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828648.456098:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828648.456101:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2083 00000100:00080000:1.0:1787828648.456104:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828648.456106:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828648.456109:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:1.0:1787828648.456116:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7480 x1874672085652608/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828648.456122:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828648.456124:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828648.456125:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828648.456129:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2083 00000800:00000100:3.0:1787828648.456133:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787828648.456134:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828648.456140:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828648.456143:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:1.0:1787828648.456153:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5e00 x1874672085652736/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828648.456160:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828648.456162:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828648.456164:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828648.456168:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2083 00000100:00080000:1.0:1787828648.456170:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1787828648.456174:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828648/real 1787828648] req@ffff9e35da6f4b40 x1874672085652224/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828673 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828648.456175:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828648.456182:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828648.456193:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6d00 x1874672085652864/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787828648.456198:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4b40 x1874672085652224/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828673 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828648.456203:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828648.456210:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787828648.456213:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828648.456219:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828648.456220:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2083 00000100:00080000:1.0:1787828648.456225:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828648.456229:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:1.0:1787828648.456232:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787828648.456236:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085652224:192.168.203.141@tcp:250:kworker.0 00000008:00100000:1.0:1787828648.456237:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828648.456251:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4f00 x1874672085652992/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828648.456257:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828648.456259:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828648.456261:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1787828648.456264:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085652352:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787828648.456268:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2083 00000100:00080000:1.0:1787828648.456272:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828648.456276:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828648.456279:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:1.0:1787828648.456287:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7840 x1874672085653120/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787828648.456292:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787828648.456303:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828648.456305:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:3.0:1787828648.456308:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828648/real 1787828648] req@ffff9e35da6f43c0 x1874672085652352/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828673 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828648.456309:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828648.456317:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2083 00000100:00080000:1.0:1787828648.456323:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828648.456328:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828648.456331:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:3.0:1787828648.456332:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f43c0 x1874672085652352/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828673 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828648.456341:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1787828648.456342:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4780 x1874672085653248/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828648.456348:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828648.456356:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085652352:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787828648.456365:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085652480:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787828648.456381:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828648.456389:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828648/real 1787828648] req@ffff9e35da6f4000 x1874672085652480/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828673 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828648.456396:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4000 x1874672085652480/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828673 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828648.456402:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828648.456403:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828648.456406:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085652480:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787828648.456409:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085652608:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787828648.456420:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828648.456426:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828648/real 1787828648] req@ffff9e35da6f7480 x1874672085652608/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828673 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828648.456433:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7480 x1874672085652608/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828673 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828648.456438:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828648.456440:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828648.456442:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085652608:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787828648.456445:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085652736:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787828648.456454:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828648.456460:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828648/real 1787828648] req@ffff9e35da6f5e00 x1874672085652736/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828673 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828648.456475:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5e00 x1874672085652736/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828673 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828648.456481:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828648.456483:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828648.456485:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085652736:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787828648.456489:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085652864:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787828648.456509:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828648.456521:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828648/real 1787828648] req@ffff9e35da6f6d00 x1874672085652864/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828673 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828648.456541:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6d00 x1874672085652864/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828673 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828648.456549:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828648.456551:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828648.456553:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085652864:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787828648.456558:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085652992:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787828648.456573:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828648.456580:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828648/real 1787828648] req@ffff9e35da6f4f00 x1874672085652992/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828673 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828648.456588:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4f00 x1874672085652992/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828673 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828648.456593:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828648.456594:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828648.456596:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085652992:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787828648.456615:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085653120:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787828648.456626:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828648.456632:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828648/real 1787828648] req@ffff9e35da6f7840 x1874672085653120/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828673 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828648.456639:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7840 x1874672085653120/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828673 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828648.456648:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828648.456650:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828648.456652:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085653120:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787828648.456656:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085653248:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787828648.456680:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828648.456688:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828648/real 1787828648] req@ffff9e35da6f4780 x1874672085653248/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828673 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828648.456696:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4780 x1874672085653248/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828673 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828648.456701:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828648.456703:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828648.456705:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085653248:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787828648.583888:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787828648.583888:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828648.583931:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787828648.583959:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828649.607934:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787828649.607977:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828649.607992:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:0.0:1787828649.607999:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828650.631923:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787828650.631923:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828650.631939:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:1.0:1787828650.632004:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828651.655970:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828651.655970:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787828651.655971:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828651.656021:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828652.679885:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787828652.679892:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787828652.679893:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787828652.679978:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828653.575870:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828653.575878:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828653.575882:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828653.575891:0:27606:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2088 00000100:00080000:1.0:1787828653.575912:0:27606:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828653.575920:0:27606:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787828653.575956:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f61c0 x1874672085653376/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828653.575980:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828653.575983:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828653.575984:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828653.575988:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2088 00000100:00080000:1.0:1787828653.575991:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828653.575993:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828653.576001:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828653.576011:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f70c0 x1874672085653504/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828653.576020:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1787828653.576023:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085653376:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787828653.576024:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787828653.576027:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828653.576035:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2088 00000100:00080000:1.0:1787828653.576038:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828653.576040:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828653.576043:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828653.576052:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6940 x1874672085653632/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828653.576062:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828653.576065:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828653.576067:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828653.576070:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2088 00000100:00080000:1.0:1787828653.576072:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828653.576075:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828653.576079:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:1.0:1787828653.576087:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6580 x1874672085653760/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828653.576092:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828653.576096:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828653.576097:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828653.576100:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2088 00000100:00080000:1.0:1787828653.576102:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828653.576104:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828653.576108:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:1.0:1787828653.576115:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5a40 x1874672085653888/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828653.576123:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1787828653.576124:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085653504:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787828653.576131:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828653.576134:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828653.576139:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2088 00000100:00080000:1.0:1787828653.576142:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828653.576145:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828653.576149:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828653.576156:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085653632:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828653.576161:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4780 x1874672085654016/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828653.576169:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828653.576172:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787828653.576174:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1787828653.576178:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085653760:192.168.203.141@tcp:8:kworker.0 00000100:00080000:1.0:1787828653.576179:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2088 00000100:00080000:1.0:1787828653.576185:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828653.576189:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828653.576193:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828653.576204:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7840 x1874672085654144/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787828653.576206:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085653888:192.168.203.141@tcp:8:kworker.0 00000100:00080000:1.0:1787828653.576221:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828653.576224:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828653.576226:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828653.576231:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2088 00000100:00100000:3.0:1787828653.576233:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085654016:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787828653.576235:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828653.576242:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828653.576246:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:1.0:1787828653.576257:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4f00 x1874672085654272/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828653.576268:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1787828653.576269:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085654144:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787828653.576274:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828653.576278:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828653.576282:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2088 00000100:00080000:1.0:1787828653.576286:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828653.576289:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828653.576293:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:3.0:1787828653.576300:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085654272:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828653.576305:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6d00 x1874672085654400/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787828653.576324:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085654400:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787828653.703910:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828653.703910:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787828653.703958:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787828653.703991:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828654.727867:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787828654.727891:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828654.727891:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828654.727941:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828655.751900:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828655.751900:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828655.751927:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787828655.751959:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1787828656.648065:0:2399:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.141:988 00000400:00000100:2.0:1787828656.648099:0:2399:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.141@tcp at host 192.168.203.141 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1787828656.648112:0:2399:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:2.0:1787828656.648147:0:2399:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000800:00000100:2.0:1787828656.648156:0:2399:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:2.0:1787828656.648161:0:2399:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:2.0:1787828656.648166:0:2399:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:2.0:1787828656.648171:0:2399:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:2.0:1787828656.648176:0:2399:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:2.0:1787828656.648180:0:2399:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:2.0:1787828656.648185:0:2399:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:3.0:1787828656.648267:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828653/real 1787828656] req@ffff9e35da6f6d00 x1874672085654400/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828678 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828656.648310:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6d00 x1874672085654400/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828678 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828656.648325:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828656.648332:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828656.648338:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085654400:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787828656.648349:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828653/real 1787828656] req@ffff9e35da6f4f00 x1874672085654272/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828678 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828656.648364:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4f00 x1874672085654272/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828678 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828656.648374:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828656.648377:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828656.648380:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085654272:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787828656.648389:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828653/real 1787828656] req@ffff9e35da6f7840 x1874672085654144/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828678 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828656.648409:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7840 x1874672085654144/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828678 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828656.648416:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828656.648420:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828656.648424:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085654144:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787828656.648431:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828653/real 1787828656] req@ffff9e35da6f5a40 x1874672085653888/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828678 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828656.648443:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5a40 x1874672085653888/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828678 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828656.648451:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828656.648455:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828656.648458:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085653888:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787828656.648465:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828653/real 1787828656] req@ffff9e35da6f4780 x1874672085654016/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828678 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828656.648477:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4780 x1874672085654016/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828678 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828656.648486:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828656.648490:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828656.648493:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085654016:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787828656.648500:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828653/real 1787828656] req@ffff9e35da6f70c0 x1874672085653504/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828678 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828656.648513:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f70c0 x1874672085653504/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828678 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828656.648522:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828656.648532:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828656.648536:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085653504:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787828656.648544:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828653/real 1787828656] req@ffff9e35da6f6940 x1874672085653632/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828678 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828656.648560:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6940 x1874672085653632/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828678 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828656.648568:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828656.648572:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828656.648575:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085653632:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787828656.648584:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828653/real 1787828656] req@ffff9e35da6f6580 x1874672085653760/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828678 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828656.648598:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6580 x1874672085653760/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828678 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828656.648621:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828656.648627:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828656.648630:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085653760:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787828656.648651:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828653/real 1787828656] req@ffff9e35da6f61c0 x1874672085653376/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828678 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828656.648667:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f61c0 x1874672085653376/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828678 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828656.648675:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828656.648679:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828656.648683:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085653376:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1787828656.775910:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828656.775928:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828656.775929:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828656.775992:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828657.799886:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828657.799887:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828657.799916:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828657.799938:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828658.695924:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828658.695934:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828658.695938:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828658.695957:0:27606:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2094 00000100:00080000:0.0:1787828658.695969:0:27606:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828658.695977:0:27606:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787828658.696020:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7192c0 x1874672085654528/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828658.696059:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828658.696063:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828658.696066:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828658.696070:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2094 00000100:00080000:0.0:1787828658.696074:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828658.696078:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828658.696093:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828658.696112:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718b40 x1874672085654656/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828658.696125:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828658.696128:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787828658.696132:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1787828658.696135:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085654528:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1787828658.696140:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2094 00000100:00080000:0.0:1787828658.696161:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828658.696166:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828658.696172:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828658.696186:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718f00 x1874672085654784/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828658.696197:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828658.696202:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787828658.696204:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828658.696209:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2094 00000100:00080000:0.0:1787828658.696214:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828658.696217:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828658.696224:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:0.0:1787828658.696237:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a940 x1874672085654912/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787828658.696242:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787828658.696254:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828658.696259:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787828658.696262:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828658.696269:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2094 00000100:00080000:0.0:1787828658.696283:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828658.696288:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828658.696294:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:0.0:1787828658.696323:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a1c0 x1874672085655040/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787828658.696329:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828658/real 1787828658] req@ffff9e35da7192c0 x1874672085654528/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828683 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828658.696339:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828658.696347:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828658.696350:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828658.696356:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2094 00000100:00100000:3.0:1787828658.696365:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7192c0 x1874672085654528/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828683 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828658.696366:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828658.696377:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828658.696388:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00080000:3.0:1787828658.696390:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828658.696398:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828658.696405:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7183c0 x1874672085655168/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787828658.696407:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085654528:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1787828658.696423:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828658.696427:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787828658.696429:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828658.696435:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2094 00000100:00080000:0.0:1787828658.696439:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828658.696443:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828658.696449:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828658.696453:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085654656:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787828658.696465:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a580 x1874672085655296/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828658.696477:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828658.696481:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787828658.696484:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828658.696489:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2094 00000100:00080000:0.0:1787828658.696495:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1787828658.696495:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787828658.696505:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828658.696512:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:0.0:1787828658.696525:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da719a40 x1874672085655424/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787828658.696534:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828658/real 1787828658] req@ffff9e35da718b40 x1874672085654656/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828683 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828658.696537:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828658.696548:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787828658.696552:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828658.696563:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2094 00000100:00100000:3.0:1787828658.696563:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718b40 x1874672085654656/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828683 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828658.696576:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828658.696579:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828658.696585:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:3.0:1787828658.696588:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000008:00100000:0.0:1787828658.696607:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:3.0:1787828658.696610:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085654656:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787828658.696621:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085654784:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787828658.696627:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718780 x1874672085655552/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787828658.696945:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828658.696960:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828658/real 1787828658] req@ffff9e35da718f00 x1874672085654784/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828683 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828658.696972:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672085654784/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828683 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828658.696981:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828658.696986:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828658.696989:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085654784:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787828658.696995:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085654912:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787828658.697019:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828658.697032:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828658/real 1787828658] req@ffff9e35da71a940 x1874672085654912/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828683 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828658.697043:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672085654912/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828683 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828658.697052:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828658.697056:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828658.697060:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085654912:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787828658.697065:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085655040:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787828658.697085:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828658.697095:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828658/real 1787828658] req@ffff9e35da71a1c0 x1874672085655040/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828683 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828658.697108:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a1c0 x1874672085655040/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828683 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828658.697118:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828658.697122:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828658.697125:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085655040:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787828658.697131:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085655168:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787828658.697152:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828658.697165:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828658/real 1787828658] req@ffff9e35da7183c0 x1874672085655168/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828683 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828658.697176:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672085655168/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828683 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828658.697186:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828658.697189:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828658.697192:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085655168:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787828658.697221:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085655296:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787828658.697247:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828658.697265:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828658/real 1787828658] req@ffff9e35da71a580 x1874672085655296/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828683 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828658.697280:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672085655296/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828683 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828658.697290:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828658.697295:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828658.697299:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085655296:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787828658.697310:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085655424:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787828658.697339:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828658.697353:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828658/real 1787828658] req@ffff9e35da719a40 x1874672085655424/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828683 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828658.697365:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672085655424/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828683 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828658.697374:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828658.697377:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828658.697381:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085655424:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787828658.697387:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085655552:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787828658.697406:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828658.697419:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828658/real 1787828658] req@ffff9e35da718780 x1874672085655552/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828683 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828658.697432:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718780 x1874672085655552/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828683 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828658.697442:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828658.697445:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828658.697449:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085655552:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828658.823962:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828658.823963:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787828658.824012:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787828658.824063:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828659.847894:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828659.847928:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787828659.847939:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828659.847963:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828660.871900:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828660.871936:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828660.871936:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828660.871957:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787828661.895884:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:3.0:1787828661.895884:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828661.895890:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787828661.895961:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828662.919904:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828662.919904:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828662.919941:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828662.919950:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828663.816007:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828663.816018:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828663.816022:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828663.816035:0:27606:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2099 00000100:00080000:1.0:1787828663.816047:0:27606:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828663.816055:0:27606:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787828663.816103:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5e00 x1874672085655680/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828663.816136:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828663.816139:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828663.816141:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828663.816144:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2099 00000100:00080000:1.0:1787828663.816147:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828663.816149:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828663.816160:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828663.816170:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7480 x1874672085655808/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828663.816178:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828663.816181:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787828663.816183:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828663.816187:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2099 00000100:00080000:1.0:1787828663.816189:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828663.816191:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828663.816194:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828663.816207:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4000 x1874672085655936/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787828663.816207:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085655680:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787828663.816233:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828663.816236:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828663.816237:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828663.816243:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2099 00000100:00080000:1.0:1787828663.816247:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828663.816250:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828663.816254:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:1.0:1787828663.816263:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f43c0 x1874672085656064/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828663.816270:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828663.816273:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828663.816274:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828663.816278:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2099 00000100:00080000:1.0:1787828663.816283:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828663.816286:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828663.816289:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000800:00000100:3.0:1787828663.816292:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828663.816306:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4b40 x1874672085656192/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828663.816315:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828663.816318:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828663.816321:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828663.816325:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2099 00000100:00080000:1.0:1787828663.816328:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828663.816333:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787828663.816338:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828663/real 1787828663] req@ffff9e35da6f5e00 x1874672085655680/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828688 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1787828663.816339:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828663.816358:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f61c0 x1874672085656320/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787828663.816376:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5e00 x1874672085655680/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828688 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828663.816401:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828663.816410:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787828663.816416:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828663.816425:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2099 00000100:00080000:1.0:1787828663.816430:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828663.816433:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828663.816434:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:3.0:1787828663.816448:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000008:00100000:1.0:1787828663.816450:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828663.816455:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085655680:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1787828663.816465:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6580 x1874672085656448/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787828663.816470:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085655808:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787828663.816496:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828663.816500:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828663.816501:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828663.816508:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2099 00000100:00080000:1.0:1787828663.816513:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828663.816519:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:3.0:1787828663.816520:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:1.0:1787828663.816532:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:3.0:1787828663.816542:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828663/real 1787828663] req@ffff9e35da6f7480 x1874672085655808/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828688 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828663.816548:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6940 x1874672085656576/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787828663.816593:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7480 x1874672085655808/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828688 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828663.816617:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828663.816626:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828663.816630:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828663.816635:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828663.816636:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2099 00000100:00080000:3.0:1787828663.816661:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:1.0:1787828663.816662:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1787828663.816671:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085655808:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787828663.816673:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828663.816689:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:1.0:1787828663.816706:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f70c0 x1874672085656704/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787828663.816724:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085655936:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787828663.816758:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828663.816774:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828663/real 1787828663] req@ffff9e35da6f4000 x1874672085655936/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828688 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828663.816783:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4000 x1874672085655936/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828688 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828663.816789:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828663.816792:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828663.816794:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085655936:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787828663.816824:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085656064:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787828663.816843:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828663.816851:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828663/real 1787828663] req@ffff9e35da6f43c0 x1874672085656064/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828688 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828663.816859:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f43c0 x1874672085656064/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828688 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828663.816865:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828663.816867:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828663.816870:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085656064:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787828663.816874:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085656192:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787828663.816885:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828663.816891:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828663/real 1787828663] req@ffff9e35da6f4b40 x1874672085656192/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828688 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828663.816898:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4b40 x1874672085656192/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828688 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828663.816904:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828663.816906:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828663.816908:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085656192:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787828663.816911:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085656320:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787828663.816922:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828663.816928:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828663/real 1787828663] req@ffff9e35da6f61c0 x1874672085656320/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828688 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828663.816935:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f61c0 x1874672085656320/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828688 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828663.816951:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828663.816953:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828663.816955:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085656320:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787828663.816959:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085656448:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787828663.816968:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828663.816975:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828663/real 1787828663] req@ffff9e35da6f6580 x1874672085656448/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828688 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828663.816982:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6580 x1874672085656448/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828688 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828663.816987:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828663.816989:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828663.816991:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085656448:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787828663.816994:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085656576:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787828663.817004:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828663.817015:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828663/real 1787828663] req@ffff9e35da6f6940 x1874672085656576/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828688 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828663.817027:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6940 x1874672085656576/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828688 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828663.817036:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828663.817039:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828663.817043:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085656576:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787828663.817050:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085656704:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787828663.817069:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828663.817094:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828663/real 1787828663] req@ffff9e35da6f70c0 x1874672085656704/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828688 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828663.817112:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f70c0 x1874672085656704/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828688 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828663.817131:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828663.817133:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828663.817136:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085656704:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828663.943859:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787828663.943927:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828663.943927:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828663.943962:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787828664.967944:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828664.967954:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828664.967967:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:0.0:1787828664.967994:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828665.991912:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787828665.991947:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828665.991956:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787828665.991973:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828667.015913:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828667.015913:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828667.015955:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828667.016030:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828668.039908:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787828668.039916:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787828668.039942:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:1.0:1787828668.039958:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828668.935923:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828668.935934:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828668.935939:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828668.935949:0:27606:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2104 00000100:00080000:2.0:1787828668.935961:0:27606:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828668.935968:0:27606:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787828668.936007:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498d680 x1874672085656832/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828668.936054:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828668.936063:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828668.936065:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828668.936070:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2104 00000100:00080000:2.0:1787828668.936075:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828668.936080:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828668.936091:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828668.936099:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085656832:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1787828668.936113:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c780 x1874672085656960/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828668.936127:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828668.936131:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787828668.936133:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828668.936138:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2104 00000100:00080000:2.0:1787828668.936144:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828668.936166:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828668.936172:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1787828668.936175:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828668.936193:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e940 x1874672085657088/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787828668.936231:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828668/real 1787828668] req@ffff9e35c498d680 x1874672085656832/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828693 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828668.936238:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828668.936245:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787828668.936248:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1787828668.936252:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498d680 x1874672085656832/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828693 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828668.936257:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2104 00000100:00080000:3.0:1787828668.936265:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828668.936267:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828668.936271:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:2.0:1787828668.936294:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787828668.936295:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085656832:192.168.203.141@tcp:250:kworker.0 00000008:00100000:2.0:1787828668.936309:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:3.0:1787828668.936326:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085656960:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828668.936332:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c3c0 x1874672085657216/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828668.936350:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:3.0:1787828668.936356:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1787828668.936361:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787828668.936365:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828668.936374:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2104 00000100:00100000:3.0:1787828668.936376:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828668/real 1787828668] req@ffff9e35c498c780 x1874672085656960/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828693 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828668.936382:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828668.936393:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787828668.936394:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c780 x1874672085656960/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828693 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1787828668.936403:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00080000:3.0:1787828668.936417:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828668.936421:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828668.936428:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085656960:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828668.936429:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498ed00 x1874672085657344/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787828668.936437:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085657088:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1787828668.936454:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828668.936459:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000800:00000100:3.0:1787828668.936463:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1787828668.936465:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828668.936475:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2104 00000100:00100000:3.0:1787828668.936477:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828668/real 1787828668] req@ffff9e35c498e940 x1874672085657088/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828693 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828668.936484:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1787828668.936493:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e940 x1874672085657088/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828693 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828668.936496:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828668.936510:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00080000:3.0:1787828668.936510:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828668.936517:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828668.936536:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085657088:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828668.936565:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498de00 x1874672085657472/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787828668.936574:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085657216:192.168.203.141@tcp:8:kworker.0 00000100:00080000:2.0:1787828668.936587:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828668.936591:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787828668.936593:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828668.936599:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2104 00000800:00000100:3.0:1787828668.936600:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828668.936611:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828668/real 1787828668] req@ffff9e35c498c3c0 x1874672085657216/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828693 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828668.936613:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828668.936622:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787828668.936626:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c3c0 x1874672085657216/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828693 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1787828668.936629:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00080000:3.0:1787828668.936638:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828668.936642:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828668.936646:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085657216:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828668.936651:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498fc00 x1874672085657600/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828668.936662:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1787828668.936663:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085657344:192.168.203.141@tcp:8:kworker.0 00000100:00080000:2.0:1787828668.936672:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787828668.936675:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828668.936680:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2104 00000100:00080000:2.0:1787828668.936685:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1787828668.936687:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1787828668.936693:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787828668.936698:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828668/real 1787828668] req@ffff9e35c498ed00 x1874672085657344/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828693 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1787828668.936699:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:3.0:1787828668.936712:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498ed00 x1874672085657344/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828693 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828668.936717:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e940 x1874672085657728/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828668.936722:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828668.936731:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828668.936736:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085657344:192.168.203.141@tcp:8:kworker.0 00000100:00080000:2.0:1787828668.936737:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828668.936744:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787828668.936746:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828668.936751:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2104 00000100:00100000:3.0:1787828668.936754:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085657472:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1787828668.936757:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828668.936764:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828668.936769:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000800:00000100:3.0:1787828668.936774:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828668.936784:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828668/real 1787828668] req@ffff9e35c498de00 x1874672085657472/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828693 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828668.936788:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c780 x1874672085657856/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787828668.936825:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498de00 x1874672085657472/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828693 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828668.936832:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828668.936838:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828668.936842:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085657472:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787828668.936851:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085657600:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787828668.936867:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828668.936877:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828668/real 1787828668] req@ffff9e35c498fc00 x1874672085657600/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828693 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828668.936885:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498fc00 x1874672085657600/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828693 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828668.936890:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828668.936892:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828668.936894:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085657600:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787828668.936898:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085657728:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787828668.936907:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828668.936913:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828668/real 1787828668] req@ffff9e35c498e940 x1874672085657728/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828693 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828668.936920:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e940 x1874672085657728/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828693 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828668.936927:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828668.936931:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828668.936934:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085657728:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787828668.936950:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085657856:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787828668.936970:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828668.936982:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828668/real 1787828668] req@ffff9e35c498c780 x1874672085657856/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828693 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828668.936991:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c780 x1874672085657856/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828693 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828668.936997:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828668.936999:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828668.937001:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085657856:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787828669.063965:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787828669.063965:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828669.063997:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:0.0:1787828669.064075:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828670.087881:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828670.087889:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828670.087911:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828670.088026:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787828671.111948:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828671.111954:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787828671.111994:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828671.111995:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828672.135916:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787828672.135918:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828672.135919:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787828672.135980:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828673.159946:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828673.159984:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787828673.159986:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828673.160074:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828674.055921:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828674.055928:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828674.055932:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828674.055940:0:27606:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2109 00000100:00080000:1.0:1787828674.055949:0:27606:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828674.055956:0:27606:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787828674.055986:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4780 x1874672085657984/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828674.056006:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828674.056010:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828674.056012:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828674.056016:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2109 00000100:00080000:1.0:1787828674.056019:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828674.056022:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787828674.056029:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085657984:192.168.203.141@tcp:250:kworker.0 00000008:00100000:1.0:1787828674.056030:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828674.056047:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5a40 x1874672085658112/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828674.056055:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828674.056058:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787828674.056059:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828674.056064:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2109 00000100:00080000:1.0:1787828674.056067:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828674.056069:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828674.056074:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828674.056082:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7840 x1874672085658240/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828674.056089:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828674.056092:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828674.056093:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828674.056096:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2109 00000800:00000100:2.0:1787828674.056100:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787828674.056101:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828674.056106:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828674.056110:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:1.0:1787828674.056119:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4f00 x1874672085658368/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828674.056126:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828674.056129:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828674.056131:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828674.056134:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2109 00000100:00080000:1.0:1787828674.056137:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787828674.056138:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828674/real 1787828674] req@ffff9e35da6f4780 x1874672085657984/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828699 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828674.056144:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828674.056152:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:1.0:1787828674.056164:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6d00 x1874672085658496/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828674.056170:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4780 x1874672085657984/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828699 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828674.056173:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828674.056182:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828674.056186:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828674.056188:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828674.056190:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2109 00000100:00080000:1.0:1787828674.056194:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828674.056195:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:1.0:1787828674.056200:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828674.056205:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828674.056207:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085657984:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1787828674.056213:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f70c0 x1874672085658624/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828674.056221:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828674.056223:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787828674.056224:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828674.056227:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2109 00000100:00080000:1.0:1787828674.056229:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828674.056232:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828674.056235:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828674.056239:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085658112:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828674.056244:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6940 x1874672085658752/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828674.056252:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828674.056254:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828674.056256:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828674.056259:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2109 00000100:00080000:1.0:1787828674.056262:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828674.056265:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828674.056269:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000800:00000100:2.0:1787828674.056272:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828674.056281:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6580 x1874672085658880/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828674.056288:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828674/real 1787828674] req@ffff9e35da6f5a40 x1874672085658112/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828699 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828674.056291:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828674.056299:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828674.056303:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828674.056308:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2109 00000100:00100000:2.0:1787828674.056312:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5a40 x1874672085658112/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828699 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828674.056313:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828674.056321:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828674.056327:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00080000:2.0:1787828674.056329:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828674.056334:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828674.056338:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f61c0 x1874672085659008/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828674.056341:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085658112:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828674.056355:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085658240:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828674.056374:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828674.056384:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828674/real 1787828674] req@ffff9e35da6f7840 x1874672085658240/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828699 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828674.056393:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7840 x1874672085658240/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828699 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828674.056400:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828674.056403:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828674.056406:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085658240:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828674.056411:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085658368:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828674.056425:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828674.056437:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828674/real 1787828674] req@ffff9e35da6f4f00 x1874672085658368/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828699 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828674.056447:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4f00 x1874672085658368/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828699 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828674.056452:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828674.056455:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828674.056457:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085658368:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828674.056462:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085658496:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828674.056473:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828674.056479:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828674/real 1787828674] req@ffff9e35da6f6d00 x1874672085658496/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828699 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828674.056502:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6d00 x1874672085658496/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828699 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828674.056512:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828674.056514:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828674.056516:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085658496:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828674.056520:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085658624:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828674.056532:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828674.056538:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828674/real 1787828674] req@ffff9e35da6f70c0 x1874672085658624/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828699 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828674.056546:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f70c0 x1874672085658624/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828699 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828674.056551:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828674.056553:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828674.056555:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085658624:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828674.056573:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085658752:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828674.056583:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828674.056589:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828674/real 1787828674] req@ffff9e35da6f6940 x1874672085658752/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828699 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828674.056596:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6940 x1874672085658752/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828699 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828674.056600:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828674.056602:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828674.056605:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085658752:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828674.056609:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085658880:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828674.056618:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828674.056625:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828674/real 1787828674] req@ffff9e35da6f6580 x1874672085658880/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828699 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828674.056637:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6580 x1874672085658880/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828699 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828674.056645:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828674.056648:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828674.056652:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085658880:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828674.056657:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085659008:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828674.056675:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828674.056687:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828674/real 1787828674] req@ffff9e35da6f61c0 x1874672085659008/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828699 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828674.056700:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f61c0 x1874672085659008/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828699 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828674.056708:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828674.056711:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828674.056715:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085659008:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828674.183851:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787828674.183871:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828674.183887:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828674.183948:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828675.207930:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787828675.207935:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828675.207974:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:0.0:1787828675.208004:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828676.231878:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787828676.231881:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787828676.231908:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828676.231917:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828677.255889:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828677.255904:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828677.255912:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828677.255970:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828678.279878:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828678.279885:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787828678.279931:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828678.279950:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00080000:1.0:1787828679.175912:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828679.175922:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828679.175927:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828679.175937:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2114 00000100:00080000:1.0:1787828679.175949:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828679.175957:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787828679.175994:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4b40 x1874672085659136/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828679.176019:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828679.176021:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828679.176023:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828679.176026:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2114 00000100:00080000:1.0:1787828679.176029:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828679.176032:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828679.176040:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828679.176050:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f43c0 x1874672085659264/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828679.176068:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828679.176071:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787828679.176073:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828679.176077:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2114 00000100:00080000:1.0:1787828679.176081:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787828679.176081:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085659136:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787828679.176084:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828679.176091:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828679.176099:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4000 x1874672085659392/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828679.176106:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828679.176109:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828679.176110:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828679.176112:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2114 00000100:00080000:1.0:1787828679.176114:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828679.176117:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828679.176121:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:1.0:1787828679.176128:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7480 x1874672085659520/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828679.176134:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828679.176137:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828679.176138:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828679.176141:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2114 00000100:00080000:1.0:1787828679.176143:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828679.176145:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828679.176148:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000800:00000100:2.0:1787828679.176155:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828679.176157:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5e00 x1874672085659648/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828679.176165:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828679.176168:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828679.176169:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828679.176173:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2114 00000100:00080000:1.0:1787828679.176178:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828679.176180:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828679.176183:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828679.176191:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f61c0 x1874672085659776/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828679.176197:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828679.176199:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787828679.176200:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828679.176202:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2114 00000100:00080000:1.0:1787828679.176204:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828679.176207:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828679.176211:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828679.176221:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828679/real 1787828679] req@ffff9e35da6f4b40 x1874672085659136/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828704 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828679.176225:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6580 x1874672085659904/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828679.176255:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828679.176259:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828679.176261:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787828679.176267:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4b40 x1874672085659136/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828704 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828679.176271:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2114 00000100:00080000:1.0:1787828679.176279:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828679.176284:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:2.0:1787828679.176287:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:1.0:1787828679.176289:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00080000:2.0:1787828679.176296:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828679.176302:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6940 x1874672085660032/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828679.176304:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085659136:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787828679.176318:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828679.176322:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:2.0:1787828679.176324:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085659264:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787828679.176325:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828679.176333:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2114 00000100:00080000:1.0:1787828679.176335:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828679.176337:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828679.176340:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:1.0:1787828679.176349:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f70c0 x1874672085660160/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787828679.176364:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828679.176382:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828679/real 1787828679] req@ffff9e35da6f43c0 x1874672085659264/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828704 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828679.176396:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f43c0 x1874672085659264/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828704 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828679.176406:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828679.176409:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828679.176413:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085659264:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828679.176438:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085659392:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828679.176478:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828679.176490:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828679/real 1787828679] req@ffff9e35da6f4000 x1874672085659392/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828704 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828679.176504:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4000 x1874672085659392/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828704 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828679.176513:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828679.176517:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828679.176519:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085659392:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828679.176524:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085659520:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828679.176539:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828679.176547:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828679/real 1787828679] req@ffff9e35da6f7480 x1874672085659520/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828704 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828679.176555:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7480 x1874672085659520/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828704 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828679.176561:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828679.176564:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828679.176566:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085659520:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828679.176569:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085659648:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828679.176580:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828679.176586:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828679/real 1787828679] req@ffff9e35da6f5e00 x1874672085659648/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828704 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828679.176598:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5e00 x1874672085659648/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828704 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828679.176604:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828679.176606:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828679.176607:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085659648:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828679.176611:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085659776:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828679.176621:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828679.176627:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828679/real 1787828679] req@ffff9e35da6f61c0 x1874672085659776/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828704 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828679.176634:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f61c0 x1874672085659776/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828704 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828679.176639:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828679.176641:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828679.176643:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085659776:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828679.176647:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085659904:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828679.176656:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828679.176662:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828679/real 1787828679] req@ffff9e35da6f6580 x1874672085659904/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828704 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828679.176669:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6580 x1874672085659904/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828704 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828679.176678:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828679.176681:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828679.176683:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085659904:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828679.176686:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085660032:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828679.176696:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828679.176702:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828679/real 1787828679] req@ffff9e35da6f6940 x1874672085660032/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828704 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828679.176708:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6940 x1874672085660032/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828704 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828679.176713:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828679.176715:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828679.176717:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085660032:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828679.176720:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085660160:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828679.176729:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828679.176734:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828679/real 1787828679] req@ffff9e35da6f70c0 x1874672085660160/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828704 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828679.176741:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f70c0 x1874672085660160/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828704 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828679.176747:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828679.176748:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828679.176751:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085660160:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828679.303887:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828679.303887:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787828679.303895:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828679.303965:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828680.327905:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828680.327912:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828680.327953:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828680.327957:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828681.351913:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828681.351926:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787828681.351971:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828681.351995:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828682.375908:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828682.375909:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828682.375938:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828682.375948:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828683.399862:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828683.399876:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:3.0:1787828683.399889:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828683.399953:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828684.295947:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828684.295960:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828684.295965:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828684.295977:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2119 00000100:00080000:0.0:1787828684.295989:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828684.295998:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787828684.296053:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718000 x1874672085660288/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828684.296092:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828684.296103:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828684.296107:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828684.296115:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2119 00000100:00080000:0.0:1787828684.296122:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828684.296131:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828684.296171:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828684.296183:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085660288:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1787828684.296201:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718780 x1874672085660416/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828684.296219:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828684.296224:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787828684.296226:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828684.296232:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2119 00000100:00080000:0.0:1787828684.296237:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828684.296242:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828684.296249:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828684.296264:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da719a40 x1874672085660544/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828684.296275:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828684.296280:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787828684.296285:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1787828684.296289:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787828684.296322:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2119 00000100:00080000:0.0:1787828684.296331:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828684.296337:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828684.296346:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:0.0:1787828684.296362:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a580 x1874672085660672/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828684.296375:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828684.296389:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787828684.296392:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828684.296398:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2119 00000100:00080000:0.0:1787828684.296405:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787828684.296409:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828684/real 1787828684] req@ffff9e35da718000 x1874672085660288/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828709 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828684.296455:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828684.296470:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:0.0:1787828684.296493:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7183c0 x1874672085660800/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828684.296508:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718000 x1874672085660288/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828709 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828684.296511:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828684.296524:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828684.296528:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828684.296542:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2119 00000100:00080000:0.0:1787828684.296552:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828684.296559:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828684.296562:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828684.296577:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1787828684.296582:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828684.296594:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a1c0 x1874672085660928/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828684.296597:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085660288:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1787828684.296622:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828684.296626:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787828684.296630:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787828684.296647:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085660416:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787828684.296648:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2119 00000100:00080000:0.0:1787828684.296659:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828684.296666:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828684.296673:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828684.296710:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a940 x1874672085661056/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787828684.296727:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787828684.296732:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828684.296739:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787828684.296742:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828684.296749:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2119 00000100:00080000:0.0:1787828684.296755:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787828684.296760:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828684/real 1787828684] req@ffff9e35da718780 x1874672085660416/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828709 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828684.296767:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828684.296779:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:0.0:1787828684.296802:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718f00 x1874672085661184/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828684.296865:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828684.296871:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787828684.296874:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828684.296879:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2119 00000100:00080000:0.0:1787828684.296885:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828684.296892:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828684.296900:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:2.0:1787828684.296906:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718780 x1874672085660416/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828709 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828684.296924:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718b40 x1874672085661312/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828684.296988:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828684.297002:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828684.297013:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085660416:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828684.297031:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085660544:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828684.297086:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828684.297115:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828684/real 1787828684] req@ffff9e35da719a40 x1874672085660544/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828709 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828684.297143:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672085660544/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828709 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828684.297159:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828684.297163:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828684.297167:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085660544:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828684.297176:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085660672:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828684.297206:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828684.297238:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828684/real 1787828684] req@ffff9e35da71a580 x1874672085660672/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828709 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828684.297253:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672085660672/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828709 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828684.297263:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828684.297267:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828684.297271:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085660672:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828684.297278:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085660800:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828684.297298:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828684.297308:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828684/real 1787828684] req@ffff9e35da7183c0 x1874672085660800/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828709 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828684.297341:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672085660800/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828709 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828684.297352:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828684.297355:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828684.297359:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085660800:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828684.297366:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085660928:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828684.297387:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828684.297399:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828684/real 1787828684] req@ffff9e35da71a1c0 x1874672085660928/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828709 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828684.297413:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a1c0 x1874672085660928/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828709 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828684.297451:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828684.297455:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828684.297459:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085660928:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828684.297509:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085661056:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828684.297534:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828684.297568:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828684/real 1787828684] req@ffff9e35da71a940 x1874672085661056/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828709 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828684.297582:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672085661056/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828709 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828684.297593:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828684.297597:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828684.297618:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085661056:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828684.297625:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085661184:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828684.297644:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828684.297654:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828684/real 1787828684] req@ffff9e35da718f00 x1874672085661184/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828709 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828684.297667:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672085661184/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828709 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828684.297677:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828684.297680:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828684.297683:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085661184:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828684.297689:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085661312:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828684.297705:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828684.297715:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828684/real 1787828684] req@ffff9e35da718b40 x1874672085661312/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828709 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828684.297728:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718b40 x1874672085661312/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828709 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828684.297738:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828684.297742:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828684.297745:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085661312:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828684.423927:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828684.424082:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787828684.424101:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:1.0:1787828684.424154:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828685.447888:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:3.0:1787828685.447888:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828685.447952:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828685.447990:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828686.471916:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828686.471916:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787828686.471922:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828686.471983:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828687.495993:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828687.495993:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787828687.496001:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:1.0:1787828687.496059:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828688.519893:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828688.519897:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787828688.519920:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828688.519981:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828689.415891:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828689.415899:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828689.415903:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828689.415911:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2124 00000100:00080000:0.0:1787828689.415921:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828689.415927:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787828689.415978:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7192c0 x1874672085661440/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828689.416012:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828689.416018:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828689.416022:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828689.416028:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2124 00000100:00080000:0.0:1787828689.416032:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828689.416037:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828689.416049:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828689.416071:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718b40 x1874672085661568/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828689.416082:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085661440:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1787828689.416093:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828689.416100:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787828689.416102:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828689.416109:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2124 00000100:00080000:0.0:1787828689.416114:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828689.416118:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828689.416124:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828689.416142:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718f00 x1874672085661696/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828689.416169:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828689.416173:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787828689.416175:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828689.416178:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2124 00000100:00080000:0.0:1787828689.416189:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828689.416192:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828689.416197:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000800:00000100:2.0:1787828689.416199:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828689.416211:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a940 x1874672085661824/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828689.416224:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828689.416227:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787828689.416228:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828689.416232:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2124 00000100:00080000:0.0:1787828689.416236:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828689.416239:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828689.416244:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:2.0:1787828689.416246:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828689/real 1787828689] req@ffff9e35da7192c0 x1874672085661440/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828714 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828689.416277:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a1c0 x1874672085661952/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828689.416290:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787828689.416293:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7192c0 x1874672085661440/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828714 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828689.416304:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828689.416310:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828689.416321:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828689.416322:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2124 00000100:00080000:0.0:1787828689.416334:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828689.416336:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828689.416345:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085661440:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1787828689.416348:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828689.416363:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828689.416401:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7183c0 x1874672085662080/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828689.416442:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787828689.416444:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085661568:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787828689.416449:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787828689.416455:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828689.416459:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2124 00000100:00080000:0.0:1787828689.416463:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828689.416465:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828689.416470:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828689.416481:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a580 x1874672085662208/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787828689.416489:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787828689.416495:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828689.416499:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787828689.416500:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828689.416505:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2124 00000100:00080000:0.0:1787828689.416510:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787828689.416511:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828689/real 1787828689] req@ffff9e35da718b40 x1874672085661568/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828714 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828689.416521:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787828689.416528:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718b40 x1874672085661568/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828714 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1787828689.416530:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00080000:2.0:1787828689.416544:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828689.416547:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828689.416553:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da719a40 x1874672085662336/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828689.416554:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085661568:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828689.416565:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085661696:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787828689.416575:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828689.416579:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787828689.416581:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1787828689.416596:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787828689.416600:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2124 00000100:00080000:0.0:1787828689.416606:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828689.416609:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828689.416614:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:2.0:1787828689.416618:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828689/real 1787828689] req@ffff9e35da718f00 x1874672085661696/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828714 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828689.416628:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718780 x1874672085662464/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828689.416647:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672085661696/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828714 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828689.416656:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828689.416659:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828689.416662:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085661696:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828689.416669:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085661824:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828689.416688:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828689.416696:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828689/real 1787828689] req@ffff9e35da71a940 x1874672085661824/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828714 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828689.416704:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672085661824/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828714 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828689.416710:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828689.416713:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828689.416716:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085661824:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828689.416720:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085661952:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828689.416731:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828689.416737:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828689/real 1787828689] req@ffff9e35da71a1c0 x1874672085661952/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828714 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828689.416745:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a1c0 x1874672085661952/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828714 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828689.416751:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828689.416753:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828689.416755:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085661952:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828689.416771:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085662080:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828689.416783:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828689.416790:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828689/real 1787828689] req@ffff9e35da7183c0 x1874672085662080/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828714 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828689.416798:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672085662080/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828714 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828689.416803:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828689.416805:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828689.416807:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085662080:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828689.416811:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085662208:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828689.416855:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828689.416864:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828689/real 1787828689] req@ffff9e35da71a580 x1874672085662208/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828714 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828689.416873:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672085662208/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828714 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828689.416879:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828689.416881:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828689.416883:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085662208:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828689.416887:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085662336:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828689.416899:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828689.416905:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828689/real 1787828689] req@ffff9e35da719a40 x1874672085662336/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828714 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828689.416913:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672085662336/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828714 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828689.416919:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828689.416921:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828689.416923:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085662336:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828689.416926:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085662464:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828689.416936:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828689.416942:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828689/real 1787828689] req@ffff9e35da718780 x1874672085662464/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828714 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828689.416950:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718780 x1874672085662464/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828714 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828689.416956:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828689.416958:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828689.416960:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085662464:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828689.543870:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787828689.543871:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828689.543912:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828689.543919:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828690.567880:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828690.567884:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828690.567915:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828690.567950:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787828691.591866:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828691.591868:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787828691.591894:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787828691.591906:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828692.615869:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787828692.615877:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787828692.615901:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828692.615911:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828693.639868:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787828693.639871:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:3.0:1787828693.639893:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828693.639962:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828694.535927:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828694.535940:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828694.535946:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828694.535959:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2129 00000100:00080000:2.0:1787828694.535972:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828694.535981:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787828694.536026:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498d680 x1874672085662592/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828694.536089:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828694.536100:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828694.536112:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828694.536124:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2129 00000100:00080000:2.0:1787828694.536134:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1787828694.536140:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085662592:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1787828694.536143:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828694.536158:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828694.536178:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cf00 x1874672085662720/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828694.536209:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:3.0:1787828694.536213:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1787828694.536221:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787828694.536226:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828694.536238:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2129 00000100:00080000:2.0:1787828694.536243:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828694.536247:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828694.536253:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828694.536259:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828694/real 1787828694] req@ffff9e35c498d680 x1874672085662592/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828719 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828694.536269:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e580 x1874672085662848/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787828694.536299:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498d680 x1874672085662592/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828719 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828694.536301:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828694.536308:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787828694.536312:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828694.536318:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828694.536319:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2129 00000100:00080000:2.0:1787828694.536325:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828694.536325:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:2.0:1787828694.536335:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828694.536365:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:3.0:1787828694.536371:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085662592:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1787828694.536399:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f0c0 x1874672085662976/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828694.536412:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828694.536416:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787828694.536420:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1787828694.536420:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085662720:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1787828694.536425:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2129 00000100:00080000:2.0:1787828694.536430:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828694.536437:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828694.536444:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000800:00000100:3.0:1787828694.536455:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828694.536459:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f480 x1874672085663104/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828694.536473:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1787828694.536474:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828694/real 1787828694] req@ffff9e35c498cf00 x1874672085662720/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828719 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828694.536479:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828694.536482:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828694.536488:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2129 00000100:00080000:2.0:1787828694.536494:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828694.536499:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787828694.536505:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498cf00 x1874672085662720/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828719 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1787828694.536508:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00080000:3.0:1787828694.536519:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828694.536524:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828694.536525:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e1c0 x1874672085663232/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787828694.536530:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085662720:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1787828694.536561:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1787828694.536563:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085662848:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1787828694.536566:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787828694.536568:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828694.536574:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2129 00000100:00080000:2.0:1787828694.536577:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828694.536583:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828694.536588:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1787828694.536594:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828694.536618:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cb40 x1874672085663360/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787828694.536629:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828694/real 1787828694] req@ffff9e35c498e580 x1874672085662848/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828719 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828694.536632:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828694.536637:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787828694.536640:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828694.536645:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2129 00000100:00100000:3.0:1787828694.536646:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e580 x1874672085662848/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828719 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828694.536650:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828694.536655:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828694.536661:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00080000:3.0:1787828694.536687:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828694.536692:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828694.536697:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085662848:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828694.536698:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c000 x1874672085663488/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787828694.536706:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085662976:192.168.203.141@tcp:8:kworker.0 00000100:00080000:2.0:1787828694.536713:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828694.536718:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787828694.536721:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828694.536726:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2129 00000100:00080000:2.0:1787828694.536731:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1787828694.536734:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1787828694.536736:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828694.536743:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:2.0:1787828694.536760:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f840 x1874672085663616/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787828694.536764:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828694/real 1787828694] req@ffff9e35c498f0c0 x1874672085662976/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828719 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828694.536785:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f0c0 x1874672085662976/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828719 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828694.536831:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828694.536853:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828694.536859:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085662976:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787828694.536890:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085663104:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787828694.536923:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828694.536942:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828694/real 1787828694] req@ffff9e35c498f480 x1874672085663104/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828719 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828694.536961:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f480 x1874672085663104/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828719 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828694.536970:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828694.536973:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828694.536979:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085663104:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787828694.536985:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085663232:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787828694.537013:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828694.537026:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828694/real 1787828694] req@ffff9e35c498e1c0 x1874672085663232/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828719 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828694.537040:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e1c0 x1874672085663232/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828719 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828694.537049:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828694.537052:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828694.537056:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085663232:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787828694.537062:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085663360:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787828694.537077:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828694.537089:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828694/real 1787828694] req@ffff9e35c498cb40 x1874672085663360/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828719 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828694.537098:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498cb40 x1874672085663360/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828719 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828694.537105:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828694.537107:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828694.537109:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085663360:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787828694.537114:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085663488:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787828694.537126:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828694.537133:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828694/real 1787828694] req@ffff9e35c498c000 x1874672085663488/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828719 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828694.537142:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c000 x1874672085663488/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828719 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828694.537148:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828694.537150:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828694.537153:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085663488:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787828694.537157:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085663616:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787828694.537167:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828694.537174:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828694/real 1787828694] req@ffff9e35c498f840 x1874672085663616/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828719 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828694.537182:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f840 x1874672085663616/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828719 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828694.537189:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828694.537191:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828694.537193:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085663616:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787828694.663946:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828694.663946:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828694.663973:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828694.664138:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828695.687865:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787828695.687898:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828695.687927:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828695.687927:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828696.711926:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787828696.711947:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828696.711955:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828696.712054:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828697.735900:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828697.735933:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828697.736005:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:3.0:1787828697.736017:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828698.759898:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828698.759898:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:3.0:1787828698.759904:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828698.759983:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828699.655951:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828699.655962:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828699.655967:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828699.655993:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2135 00000100:00080000:0.0:1787828699.656012:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828699.656018:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787828699.656075:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718000 x1874672085663744/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828699.656108:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828699.656114:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828699.656117:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828699.656124:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2135 00000100:00080000:0.0:1787828699.656146:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828699.656151:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828699.656162:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828699.656181:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718780 x1874672085663872/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787828699.656190:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085663744:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1787828699.656210:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828699.656215:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787828699.656217:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828699.656223:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2135 00000100:00080000:0.0:1787828699.656229:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828699.656234:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828699.656241:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828699.656254:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da719a40 x1874672085664000/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828699.656267:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828699.656272:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787828699.656274:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828699.656280:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2135 00000800:00000100:3.0:1787828699.656280:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787828699.656285:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828699.656291:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828699.656297:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:0.0:1787828699.656333:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a580 x1874672085664128/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828699.656349:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1787828699.656353:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828699/real 1787828699] req@ffff9e35da718000 x1874672085663744/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828724 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828699.656355:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787828699.656358:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828699.656364:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2135 00000100:00080000:0.0:1787828699.656369:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828699.656374:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787828699.656379:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718000 x1874672085663744/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828724 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1787828699.656382:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00080000:3.0:1787828699.656396:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1787828699.656399:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7183c0 x1874672085664256/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828699.656402:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828699.656409:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085663744:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1787828699.656414:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828699.656418:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828699.656422:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828699.656427:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2135 00000100:00080000:0.0:1787828699.656432:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828699.656437:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787828699.656440:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085663872:192.168.203.141@tcp:38:kworker.0 00000008:00100000:0.0:1787828699.656443:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828699.656458:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a1c0 x1874672085664384/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787828699.656475:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787828699.656485:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828699.656490:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787828699.656492:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1787828699.656496:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828699/real 1787828699] req@ffff9e35da718780 x1874672085663872/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828724 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828699.656498:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2135 00000100:00080000:0.0:1787828699.656504:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828699.656509:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828699.656516:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828699.656516:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718780 x1874672085663872/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828724 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828699.656531:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a940 x1874672085664512/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828699.656532:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828699.656538:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828699.656544:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085663872:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787828699.656546:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828699.656551:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:3.0:1787828699.656554:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085664000:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787828699.656555:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828699.656560:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2135 00000100:00080000:0.0:1787828699.656565:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828699.656570:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828699.656577:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000800:00000100:3.0:1787828699.656582:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828699.656607:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718f00 x1874672085664640/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787828699.656613:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828699/real 1787828699] req@ffff9e35da719a40 x1874672085664000/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828724 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828699.656620:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828699.656624:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787828699.656627:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828699.656632:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2135 00000100:00080000:0.0:1787828699.656637:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1787828699.656638:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672085664000/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828724 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828699.656641:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828699.656647:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00080000:3.0:1787828699.656649:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828699.656653:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828699.656657:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085664000:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787828699.656664:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085664128:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787828699.656686:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718b40 x1874672085664768/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787828699.656708:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828699.656727:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828699/real 1787828699] req@ffff9e35da71a580 x1874672085664128/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828724 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828699.656747:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672085664128/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828724 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828699.656760:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828699.656764:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828699.656768:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085664128:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787828699.656775:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085664256:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787828699.656813:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828699.656823:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828699/real 1787828699] req@ffff9e35da7183c0 x1874672085664256/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828724 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828699.656832:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672085664256/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828724 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828699.656896:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828699.656901:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828699.656904:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085664256:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787828699.656928:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085664384:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787828699.656952:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828699.656964:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828699/real 1787828699] req@ffff9e35da71a1c0 x1874672085664384/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828724 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828699.656992:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a1c0 x1874672085664384/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828724 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828699.656999:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828699.657003:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828699.657005:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085664384:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787828699.657009:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085664512:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787828699.657023:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828699.657030:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828699/real 1787828699] req@ffff9e35da71a940 x1874672085664512/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828724 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828699.657039:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672085664512/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828724 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828699.657045:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828699.657047:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828699.657051:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085664512:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787828699.657055:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085664640:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787828699.657067:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828699.657074:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828699/real 1787828699] req@ffff9e35da718f00 x1874672085664640/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828724 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828699.657082:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672085664640/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828724 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828699.657089:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828699.657091:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828699.657093:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085664640:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787828699.657098:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085664768:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787828699.657109:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828699.657116:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828699/real 1787828699] req@ffff9e35da718b40 x1874672085664768/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828724 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828699.657125:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718b40 x1874672085664768/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828724 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828699.657131:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828699.657133:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828699.657135:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085664768:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828699.783960:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828699.783964:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:3.0:1787828699.783976:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828699.784079:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828700.807916:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828700.807917:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787828700.807986:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828700.807988:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828701.832003:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828701.832003:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787828701.832055:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828701.832066:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828702.855995:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787828702.855996:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:3.0:1787828702.856096:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828702.856278:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828703.879882:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787828703.879883:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787828703.879895:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828703.879974:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828704.775905:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828704.775914:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828704.775919:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828704.775928:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2140 00000100:00080000:0.0:1787828704.775938:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828704.775944:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787828704.775981:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7192c0 x1874672085664896/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828704.776018:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828704.776021:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828704.776023:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828704.776027:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2140 00000100:00080000:0.0:1787828704.776029:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828704.776032:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828704.776042:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828704.776055:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718b40 x1874672085665024/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828704.776067:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828704.776071:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787828704.776085:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1787828704.776087:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085664896:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1787828704.776092:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2140 00000100:00080000:0.0:1787828704.776098:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828704.776101:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828704.776105:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828704.776116:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718f00 x1874672085665152/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828704.776126:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828704.776129:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787828704.776130:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828704.776134:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2140 00000100:00080000:0.0:1787828704.776137:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828704.776139:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828704.776143:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:0.0:1787828704.776171:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a940 x1874672085665280/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787828704.776180:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787828704.776189:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828704.776193:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787828704.776195:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828704.776200:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2140 00000100:00080000:0.0:1787828704.776207:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828704.776211:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828704.776215:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:0.0:1787828704.776244:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a1c0 x1874672085665408/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787828704.776254:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828704/real 1787828704] req@ffff9e35da7192c0 x1874672085664896/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828729 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828704.776257:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828704.776268:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828704.776295:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828704.776329:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2140 00000100:00080000:0.0:1787828704.776335:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1787828704.776340:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7192c0 x1874672085664896/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828729 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828704.776346:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828704.776357:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00080000:3.0:1787828704.776393:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1787828704.776405:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7183c0 x1874672085665536/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828704.776406:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828704.776420:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085664896:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1787828704.776435:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828704.776458:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787828704.776462:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828704.776469:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2140 00000100:00080000:0.0:1787828704.776476:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1787828704.776481:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085665024:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787828704.776483:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828704.776496:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828704.776533:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a580 x1874672085665664/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787828704.776554:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787828704.776569:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828704.776575:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:3.0:1787828704.776577:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828704/real 1787828704] req@ffff9e35da718b40 x1874672085665024/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828729 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828704.776579:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828704.776589:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2140 00000100:00080000:0.0:1787828704.776600:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1787828704.776611:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718b40 x1874672085665024/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828729 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828704.776621:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828704.776633:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00080000:3.0:1787828704.776637:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828704.776644:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828704.776649:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da719a40 x1874672085665792/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787828704.776653:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085665024:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787828704.776672:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085665152:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787828704.776674:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828704.776683:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787828704.776685:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828704.776690:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2140 00000100:00080000:0.0:1787828704.776697:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828704.776701:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828704.776707:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000800:00000100:3.0:1787828704.776709:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828704.776724:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718780 x1874672085665920/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787828704.776746:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828704/real 1787828704] req@ffff9e35da718f00 x1874672085665152/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828729 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828704.776778:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672085665152/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828729 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828704.776787:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828704.776790:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828704.776794:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085665152:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787828704.776801:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085665280:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787828704.776818:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828704.776866:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828704/real 1787828704] req@ffff9e35da71a940 x1874672085665280/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828729 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828704.776878:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672085665280/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828729 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828704.776896:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828704.776899:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828704.776902:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085665280:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787828704.776907:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085665408:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787828704.776924:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828704.776934:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828704/real 1787828704] req@ffff9e35da71a1c0 x1874672085665408/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828729 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828704.776944:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a1c0 x1874672085665408/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828729 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828704.776951:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828704.776954:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828704.776956:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085665408:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787828704.776961:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085665536:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787828704.776974:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828704.776981:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828704/real 1787828704] req@ffff9e35da7183c0 x1874672085665536/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828729 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828704.776991:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672085665536/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828729 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828704.776997:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828704.777000:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828704.777002:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085665536:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787828704.777030:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085665664:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787828704.777043:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828704.777051:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828704/real 1787828704] req@ffff9e35da71a580 x1874672085665664/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828729 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828704.777061:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672085665664/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828729 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828704.777073:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828704.777087:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828704.777089:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085665664:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787828704.777094:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085665792:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787828704.777106:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828704.777113:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828704/real 1787828704] req@ffff9e35da719a40 x1874672085665792/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828729 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828704.777122:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672085665792/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828729 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828704.777129:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828704.777131:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828704.777134:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085665792:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787828704.777138:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085665920:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787828704.777150:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828704.777157:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828704/real 1787828704] req@ffff9e35da718780 x1874672085665920/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828729 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828704.777165:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718780 x1874672085665920/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828729 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828704.777172:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828704.777174:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828704.777177:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085665920:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787828704.903903:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828704.903904:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828704.903932:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828704.903958:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828705.927914:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787828705.927914:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828705.927955:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828705.928055:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828706.951890:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828706.951916:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787828706.951942:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828706.951946:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828707.975885:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828707.975885:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787828707.975888:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828707.975929:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828708.999897:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:3.0:1787828708.999897:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787828708.999899:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828708.999961:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828709.895956:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828709.895967:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828709.895972:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828709.895983:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2145 00000100:00080000:0.0:1787828709.895993:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828709.896018:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787828709.896053:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718000 x1874672085666048/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828709.896077:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828709.896080:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828709.896098:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828709.896102:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2145 00000100:00080000:0.0:1787828709.896107:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828709.896112:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828709.896124:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828709.896142:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085666048:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1787828709.896147:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718780 x1874672085666176/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828709.896165:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828709.896169:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787828709.896172:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828709.896178:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2145 00000100:00080000:0.0:1787828709.896183:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828709.896188:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828709.896193:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828709.896208:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da719a40 x1874672085666304/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828709.896219:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828709.896226:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000100:2.0:1787828709.896226:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787828709.896233:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828709.896262:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2145 00000100:00080000:0.0:1787828709.896270:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828709.896287:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828709.896305:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:0.0:1787828709.896321:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a580 x1874672085666432/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828709.896333:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828709.896337:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787828709.896340:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828709.896354:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2145 00000100:00100000:2.0:1787828709.896362:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828709/real 1787828709] req@ffff9e35da718000 x1874672085666048/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828734 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828709.896363:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828709.896375:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828709.896388:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:2.0:1787828709.896404:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718000 x1874672085666048/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828734 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828709.896409:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7183c0 x1874672085666560/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828709.896443:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828709.896444:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828709.896453:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828709.896455:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:0.0:1787828709.896458:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787828709.896465:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085666048:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1787828709.896470:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2145 00000100:00080000:0.0:1787828709.896483:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828709.896505:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828709.896509:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828709.896518:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a1c0 x1874672085666688/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828709.896528:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787828709.896529:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085666176:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787828709.896532:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787828709.896542:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828709.896551:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2145 00000100:00080000:0.0:1787828709.896554:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828709.896558:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828709.896563:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828709.896592:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a940 x1874672085666816/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787828709.896603:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787828709.896616:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828709.896620:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787828709.896622:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828709.896629:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2145 00000100:00100000:2.0:1787828709.896632:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828709/real 1787828709] req@ffff9e35da718780 x1874672085666176/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828734 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828709.896636:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828709.896643:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828709.896649:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:0.0:1787828709.896681:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718f00 x1874672085666944/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828709.896687:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718780 x1874672085666176/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828734 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828709.896759:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828709.896766:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787828709.896767:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828709.896769:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828709.896778:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:0.0:1787828709.896783:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2145 00000100:00100000:2.0:1787828709.896788:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085666176:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787828709.896797:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787828709.896802:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085666304:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787828709.896811:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828709.896837:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000800:00000100:2.0:1787828709.896879:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828709.896887:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718b40 x1874672085667072/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828709.896902:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828709/real 1787828709] req@ffff9e35da719a40 x1874672085666304/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828734 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828709.896938:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672085666304/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828734 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828709.896951:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828709.896955:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828709.896958:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085666304:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828709.896965:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085666432:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828709.897012:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828709.897021:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828709/real 1787828709] req@ffff9e35da71a580 x1874672085666432/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828734 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828709.897031:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672085666432/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828734 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828709.897038:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828709.897040:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828709.897044:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085666432:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828709.897048:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085666560:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828709.897089:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828709.897097:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828709/real 1787828709] req@ffff9e35da7183c0 x1874672085666560/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828734 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828709.897108:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672085666560/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828734 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828709.897116:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828709.897118:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828709.897121:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085666560:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828709.897160:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085666688:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828709.897177:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828709.897187:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828709/real 1787828709] req@ffff9e35da71a1c0 x1874672085666688/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828734 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828709.897199:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a1c0 x1874672085666688/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828734 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828709.897207:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828709.897210:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828709.897213:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085666688:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828709.897218:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085666816:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828709.897232:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828709.897266:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828709/real 1787828709] req@ffff9e35da71a940 x1874672085666816/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828734 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828709.897279:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672085666816/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828734 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828709.897288:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828709.897290:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828709.897294:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085666816:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828709.897299:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085666944:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828709.897318:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828709.897338:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828709/real 1787828709] req@ffff9e35da718f00 x1874672085666944/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828734 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828709.897349:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672085666944/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828734 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828709.897357:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828709.897360:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828709.897363:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085666944:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828709.897368:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085667072:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828709.897382:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828709.897390:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828709/real 1787828709] req@ffff9e35da718b40 x1874672085667072/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828734 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828709.897401:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718b40 x1874672085667072/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828734 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828709.897410:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828709.897413:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828709.897416:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085667072:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828710.023918:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828710.023918:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787828710.023949:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828710.023988:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828711.047948:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828711.047954:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787828711.047955:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828711.048005:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828712.071878:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787828712.071885:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:3.0:1787828712.071891:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828712.071964:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828713.095934:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:3.0:1787828713.095945:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828713.095953:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828713.096040:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828714.119909:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787828714.119909:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787828714.120048:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828714.120071:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828715.015946:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828715.015960:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828715.015966:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828715.015977:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2150 00000100:00080000:1.0:1787828715.015989:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828715.015998:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787828715.016048:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6d00 x1874672085667200/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828715.016076:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828715.016080:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828715.016081:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828715.016085:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2150 00000100:00080000:1.0:1787828715.016088:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828715.016093:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828715.016105:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828715.016118:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4f00 x1874672085667328/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828715.016130:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828715.016134:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787828715.016140:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787828715.016142:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085667200:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787828715.016150:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2150 00000100:00080000:1.0:1787828715.016164:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828715.016169:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828715.016173:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828715.016185:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7840 x1874672085667456/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828715.016193:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828715.016197:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828715.016199:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828715.016218:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2150 00000100:00080000:1.0:1787828715.016229:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828715.016234:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:2.0:1787828715.016241:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:1.0:1787828715.016242:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:1.0:1787828715.016257:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5a40 x1874672085667584/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828715.016265:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828715.016268:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828715.016270:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828715.016277:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2150 00000100:00080000:1.0:1787828715.016280:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828715.016285:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787828715.016286:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828715/real 1787828715] req@ffff9e35da6f6d00 x1874672085667200/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828740 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1787828715.016295:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:1.0:1787828715.016313:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4780 x1874672085667712/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828715.016320:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6d00 x1874672085667200/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828740 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828715.016348:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828715.016366:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828715.016369:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828715.016371:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828715.016376:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2150 00000100:00080000:2.0:1787828715.016381:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:1.0:1787828715.016382:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787828715.016391:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085667200:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787828715.016396:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828715.016406:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828715.016418:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085667328:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828715.016427:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6d00 x1874672085667840/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787828715.016439:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787828715.016443:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828715.016446:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787828715.016448:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787828715.016456:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828715/real 1787828715] req@ffff9e35da6f4f00 x1874672085667328/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828740 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828715.016460:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2150 00000100:00080000:1.0:1787828715.016471:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828715.016483:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787828715.016486:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4f00 x1874672085667328/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828740 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1787828715.016490:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1787828715.016498:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828715.016508:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828715.016511:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085667328:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828715.016518:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f70c0 x1874672085667968/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828715.016521:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085667456:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787828715.016545:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828715.016549:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828715.016552:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1787828715.016560:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787828715.016565:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2150 00000100:00080000:1.0:1787828715.016571:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787828715.016575:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828715/real 1787828715] req@ffff9e35da6f7840 x1874672085667456/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828740 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828715.016579:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828715.016592:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:2.0:1787828715.016603:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7840 x1874672085667456/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828740 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828715.016608:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6940 x1874672085668096/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828715.016643:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828715.016654:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:1.0:1787828715.016657:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787828715.016661:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085667456:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787828715.016675:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828715.016678:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787828715.016679:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085667584:192.168.203.141@tcp:8:kworker.0 00000100:00080000:1.0:1787828715.016693:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2150 00000100:00080000:1.0:1787828715.016698:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828715.016704:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:2.0:1787828715.016708:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:1.0:1787828715.016709:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:2.0:1787828715.016717:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828715/real 1787828715] req@ffff9e35da6f5a40 x1874672085667584/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828740 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828715.016727:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6580 x1874672085668224/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828715.016732:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5a40 x1874672085667584/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828740 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828715.016748:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828715.016750:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828715.016762:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085667584:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828715.016774:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085667712:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828715.016795:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828715.016828:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828715/real 1787828715] req@ffff9e35da6f4780 x1874672085667712/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828740 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828715.016841:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4780 x1874672085667712/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828740 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828715.016850:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828715.016854:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828715.016857:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085667712:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828715.016881:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085667840:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828715.016901:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828715.016910:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828715/real 1787828715] req@ffff9e35da6f6d00 x1874672085667840/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828740 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828715.016926:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6d00 x1874672085667840/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828740 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828715.016932:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828715.016934:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828715.016937:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085667840:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828715.016941:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085667968:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828715.016953:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828715.016959:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828715/real 1787828715] req@ffff9e35da6f70c0 x1874672085667968/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828740 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828715.016972:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f70c0 x1874672085667968/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828740 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828715.016977:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828715.016979:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828715.016981:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085667968:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828715.016985:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085668096:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828715.016995:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828715.017001:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828715/real 1787828715] req@ffff9e35da6f6940 x1874672085668096/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828740 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828715.017008:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6940 x1874672085668096/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828740 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828715.017014:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828715.017016:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828715.017018:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085668096:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828715.017022:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085668224:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828715.017031:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828715.017037:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828715/real 1787828715] req@ffff9e35da6f6580 x1874672085668224/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828740 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828715.017044:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6580 x1874672085668224/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828740 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828715.017053:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828715.017055:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828715.017057:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085668224:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787828715.143882:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828715.143893:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787828715.143893:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828715.143957:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828716.167887:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828716.167913:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828716.167915:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828716.167940:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828717.191872:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787828717.191874:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787828717.191883:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828717.191939:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828718.215896:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787828718.215896:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828718.215938:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828718.215965:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828719.239915:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787828719.239932:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828719.239988:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828719.240040:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828720.135970:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828720.135977:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787828720.135981:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828720.135989:0:27606:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2155 00000100:00080000:3.0:1787828720.136000:0:27606:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828720.136005:0:27606:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787828720.136039:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649ad00 x1874672085668352/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828720.136069:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828720.136071:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828720.136073:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828720.136076:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2155 00000100:00080000:3.0:1787828720.136078:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828720.136080:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828720.136088:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828720.136097:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c64992c0 x1874672085668480/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828720.136104:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828720.136107:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787828720.136109:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787828720.136112:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085668352:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1787828720.136114:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2155 00000100:00080000:3.0:1787828720.136120:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828720.136123:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828720.136127:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828720.136145:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498000 x1874672085668608/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828720.136160:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828720.136163:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787828720.136178:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828720.136183:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2155 00000100:00080000:3.0:1787828720.136189:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828720.136193:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828720.136199:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:3.0:1787828720.136212:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a1c0 x1874672085668736/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828720.136219:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828720.136221:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787828720.136223:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828720.136226:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2155 00000100:00080000:3.0:1787828720.136228:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828720.136231:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828720.136235:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:2.0:1787828720.136237:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c64992c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085668480:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787828720.136244:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c64983c0 x1874672085668864/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828720.136256:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828720.136259:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828720.136260:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828720.136263:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2155 00000100:00080000:3.0:1787828720.136274:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828720.136281:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787828720.136282:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085668608:192.168.203.141@tcp:38:kworker.0 00000008:00100000:3.0:1787828720.136285:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828720.136299:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499e00 x1874672085668992/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828720.136307:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085668736:192.168.203.141@tcp:8:kworker.0 00000100:00080000:3.0:1787828720.136320:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828720.136322:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787828720.136324:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828720.136329:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2155 00000100:00080000:3.0:1787828720.136332:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787828720.136336:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c64983c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085668864:192.168.203.141@tcp:8:kworker.0 00000100:00080000:3.0:1787828720.136337:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828720.136344:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828720.136355:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b480 x1874672085669120/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828720.136363:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787828720.136365:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085668992:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1787828720.136367:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787828720.136371:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828720.136375:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2155 00000100:00080000:3.0:1787828720.136379:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828720.136382:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828720.136386:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:2.0:1787828720.136393:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085669120:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787828720.136398:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b0c0 x1874672085669248/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828720.136408:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828720.136410:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787828720.136412:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828720.136416:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2155 00000100:00100000:2.0:1787828720.136420:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085669248:192.168.203.141@tcp:8:kworker.0 00000100:00080000:3.0:1787828720.136421:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828720.136428:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828720.136432:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:3.0:1787828720.136443:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a580 x1874672085669376/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828720.136483:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085669376:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828720.263895:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828720.263902:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828720.263932:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:0.0:1787828720.263959:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828721.287897:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828721.287897:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828721.287934:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828721.287959:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828722.311905:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828722.311941:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828722.311948:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828722.311975:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000400:00000100:1.0:1787828723.208093:0:2396:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.141:988 00000400:00000100:1.0:1787828723.208141:0:2396:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.141@tcp at host 192.168.203.141 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1787828723.208200:0:2396:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:1.0:1787828723.208262:0:2396:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000800:00000100:1.0:1787828723.208282:0:2396:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:1.0:1787828723.208293:0:2396:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:1.0:1787828723.208312:0:2396:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:1.0:1787828723.208329:0:2396:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:1.0:1787828723.208354:0:2396:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:2.0:1787828723.208357:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828720/real 1787828723] req@ffff9e35c649a580 x1874672085669376/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828745 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:1.0:1787828723.208375:0:2396:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:1.0:1787828723.208401:0:2396:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:2.0:1787828723.208489:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a580 x1874672085669376/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828745 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828723.208510:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828723.208518:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828723.208525:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085669376:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828723.208538:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828720/real 1787828723] req@ffff9e35c649b0c0 x1874672085669248/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828745 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828723.208556:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b0c0 x1874672085669248/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828745 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828723.208569:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828723.208574:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828723.208578:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085669248:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828723.208588:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828720/real 1787828723] req@ffff9e35c649b480 x1874672085669120/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828745 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828723.208606:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b480 x1874672085669120/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828745 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828723.208613:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828723.208617:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828723.208621:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085669120:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828723.208627:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828720/real 1787828723] req@ffff9e35c64983c0 x1874672085668864/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828745 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828723.208638:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c64983c0 x1874672085668864/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828745 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828723.208645:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828723.208649:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828723.208653:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c64983c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085668864:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828723.208661:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828720/real 1787828723] req@ffff9e35c6499e00 x1874672085668992/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828745 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828723.208677:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499e00 x1874672085668992/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828745 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828723.208695:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828723.208700:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828723.208705:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085668992:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828723.208714:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828720/real 1787828723] req@ffff9e35c64992c0 x1874672085668480/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828745 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828723.208731:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c64992c0 x1874672085668480/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828745 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828723.208741:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828723.208745:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828723.208750:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c64992c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085668480:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828723.208759:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828720/real 1787828723] req@ffff9e35c6498000 x1874672085668608/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828745 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828723.208772:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498000 x1874672085668608/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828745 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828723.208782:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828723.208786:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828723.208790:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085668608:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828723.208836:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828720/real 1787828723] req@ffff9e35c649a1c0 x1874672085668736/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828745 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828723.208853:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a1c0 x1874672085668736/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828745 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828723.208864:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828723.208868:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828723.208872:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085668736:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828723.208880:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828720/real 1787828723] req@ffff9e35c649ad00 x1874672085668352/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828745 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828723.208895:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649ad00 x1874672085668352/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828745 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828723.208903:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828723.208907:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828723.208912:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085668352:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1787828723.335974:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828723.335986:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828723.336027:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787828723.336101:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828724.359940:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828724.359940:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828724.359963:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828724.359988:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00080000:3.0:1787828725.255900:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828725.255910:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787828725.255914:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828725.255923:0:27606:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2160 00000100:00080000:3.0:1787828725.255935:0:27606:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828725.255942:0:27606:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787828725.255977:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649bc00 x1874672085669504/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828725.256015:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828725.256019:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828725.256024:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828725.256028:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2160 00000100:00080000:3.0:1787828725.256032:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828725.256036:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828725.256046:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828725.256060:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b840 x1874672085669632/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828725.256070:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828725.256074:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787828725.256075:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828725.256079:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2160 00000100:00080000:3.0:1787828725.256081:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828725.256084:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828725.256087:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828725.256098:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085669504:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1787828725.256099:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498f00 x1874672085669760/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828725.256127:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828725.256145:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787828725.256147:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828725.256152:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2160 00000100:00080000:3.0:1787828725.256155:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828725.256159:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828725.256164:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:3.0:1787828725.256178:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499680 x1874672085669888/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828725.256189:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828725.256195:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787828725.256197:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828725.256202:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2160 00000100:00080000:3.0:1787828725.256210:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828725.256216:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828725.256224:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000800:00000100:2.0:1787828725.256232:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828725.256252:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498b40 x1874672085670016/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828725.256270:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828725.256275:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828725.256277:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828725.256288:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2160 00000100:00080000:3.0:1787828725.256295:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787828725.256300:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828725/real 1787828725] req@ffff9e35c649bc00 x1874672085669504/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828750 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828725.256310:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828725.256329:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828725.256336:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649bc00 x1874672085669504/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828750 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828725.256351:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828725.256357:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828725.256365:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a940 x1874672085670144/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828725.256368:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085669504:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1787828725.256399:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085669632:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1787828725.256400:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828725.256418:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787828725.256421:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828725.256431:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2160 00000100:00080000:3.0:1787828725.256439:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828725.256445:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828725.256454:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1787828725.256472:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828725.256481:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498780 x1874672085670272/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828725.256502:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828725/real 1787828725] req@ffff9e35c649b840 x1874672085669632/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828750 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828725.256521:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828725.256529:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787828725.256533:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828725.256543:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2160 00000100:00080000:3.0:1787828725.256569:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787828725.256570:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b840 x1874672085669632/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828750 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828725.256587:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:2.0:1787828725.256608:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000008:00100000:3.0:1787828725.256608:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:3.0:1787828725.256632:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499a40 x1874672085670400/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828725.256641:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828725.256645:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787828725.256647:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828725.256655:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2160 00000100:00080000:2.0:1787828725.256666:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:3.0:1787828725.256671:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787828725.256679:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085669632:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1787828725.256690:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828725.256716:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:3.0:1787828725.256746:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649ad00 x1874672085670528/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828725.256779:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085669760:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828725.256904:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828725.256928:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828725/real 1787828725] req@ffff9e35c6498f00 x1874672085669760/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828750 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828725.256944:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498f00 x1874672085669760/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828750 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828725.256955:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828725.256959:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828725.256964:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085669760:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828725.256976:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085669888:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828725.257003:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828725.257036:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828725/real 1787828725] req@ffff9e35c6499680 x1874672085669888/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828750 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828725.257055:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499680 x1874672085669888/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828750 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828725.257068:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828725.257075:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828725.257085:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085669888:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828725.257096:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085670016:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828725.257159:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828725.257176:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828725/real 1787828725] req@ffff9e35c6498b40 x1874672085670016/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828750 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828725.257192:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498b40 x1874672085670016/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828750 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828725.257203:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828725.257208:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828725.257211:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085670016:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828725.257220:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085670144:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828725.257245:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828725.257260:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828725/real 1787828725] req@ffff9e35c649a940 x1874672085670144/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828750 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828725.257276:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a940 x1874672085670144/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828750 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828725.257286:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828725.257291:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828725.257295:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085670144:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828725.257305:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085670272:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828725.257335:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828725.257349:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828725/real 1787828725] req@ffff9e35c6498780 x1874672085670272/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828750 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828725.257364:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498780 x1874672085670272/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828750 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828725.257382:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828725.257386:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828725.257390:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085670272:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828725.257399:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085670400:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828725.257426:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828725.257440:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828725/real 1787828725] req@ffff9e35c6499a40 x1874672085670400/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828750 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828725.257456:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499a40 x1874672085670400/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828750 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828725.257467:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828725.257471:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828725.257475:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085670400:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828725.257481:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085670528:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828725.257506:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828725.257521:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828725/real 1787828725] req@ffff9e35c649ad00 x1874672085670528/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828750 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828725.257535:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649ad00 x1874672085670528/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828750 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828725.257546:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828725.257550:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828725.257555:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085670528:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787828725.383928:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828725.383928:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828725.383928:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787828725.383996:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828726.407854:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787828726.407903:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828726.407908:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:0.0:1787828726.407968:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828727.431900:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828727.431901:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787828727.431965:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828727.431965:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828728.455873:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828728.455913:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828728.455917:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828728.455982:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787828729.479910:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828729.479927:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828729.479954:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828729.479969:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828730.375924:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828730.375932:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828730.375936:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828730.375945:0:27606:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2165 00000100:00080000:1.0:1787828730.375953:0:27606:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828730.375964:0:27606:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787828730.375996:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f61c0 x1874672085670656/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828730.376018:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828730.376020:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828730.376022:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828730.376025:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2165 00000100:00080000:1.0:1787828730.376027:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828730.376029:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828730.376037:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828730.376045:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5e00 x1874672085670784/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828730.376057:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828730.376061:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787828730.376063:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828730.376066:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2165 00000100:00080000:1.0:1787828730.376068:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828730.376071:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828730.376075:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828730.376081:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7480 x1874672085670912/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828730.376088:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828730.376092:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828730.376093:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828730.376114:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2165 00000100:00080000:1.0:1787828730.376119:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787828730.376119:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085670656:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787828730.376131:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828730.376137:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:1.0:1787828730.376152:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4000 x1874672085671040/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828730.376159:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828730.376162:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828730.376164:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828730.376166:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2165 00000100:00080000:1.0:1787828730.376169:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828730.376174:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828730.376178:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:1.0:1787828730.376185:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f43c0 x1874672085671168/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828730.376192:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828730.376195:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828730.376196:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828730.376199:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2165 00000100:00080000:1.0:1787828730.376202:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828730.376205:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828730.376211:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1787828730.376211:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828730.376226:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4b40 x1874672085671296/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828730.376234:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828730.376236:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787828730.376239:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828730.376242:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2165 00000100:00080000:1.0:1787828730.376245:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828730.376247:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828730.376252:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828730.376266:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6580 x1874672085671424/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828730.376274:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828730/real 1787828730] req@ffff9e35da6f61c0 x1874672085670656/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828730.376324:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828730.376330:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:2.0:1787828730.376331:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f61c0 x1874672085670656/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828730.376339:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828730.376351:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2165 00000100:00080000:1.0:1787828730.376357:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828730.376357:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828730.376367:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:2.0:1787828730.376368:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828730.376384:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085670656:192.168.203.141@tcp:250:kworker.0 00000008:00100000:1.0:1787828730.376385:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:2.0:1787828730.376407:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085670784:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828730.376414:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6940 x1874672085671552/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828730.376424:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828730.376427:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828730.376429:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828730.376434:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2165 00000800:00000100:2.0:1787828730.376443:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787828730.376445:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828730.376450:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828730.376457:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:2.0:1787828730.376463:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828730/real 1787828730] req@ffff9e35da6f5e00 x1874672085670784/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828730.376474:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f70c0 x1874672085671680/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828730.376487:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5e00 x1874672085670784/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828730.376506:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828730.376509:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828730.376513:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085670784:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828730.376549:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085670912:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828730.376563:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828730.376570:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828730/real 1787828730] req@ffff9e35da6f7480 x1874672085670912/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828730.376578:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7480 x1874672085670912/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828730.376583:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828730.376586:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828730.376588:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085670912:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828730.376592:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085671040:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828730.376603:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828730.376609:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828730/real 1787828730] req@ffff9e35da6f4000 x1874672085671040/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828730.376616:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4000 x1874672085671040/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828730.376622:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828730.376624:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828730.376625:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085671040:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828730.376629:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085671168:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828730.376639:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828730.376645:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828730/real 1787828730] req@ffff9e35da6f43c0 x1874672085671168/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828730.376653:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f43c0 x1874672085671168/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828730.376659:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828730.376661:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828730.376663:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085671168:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828730.376666:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085671296:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828730.376677:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828730.376683:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828730/real 1787828730] req@ffff9e35da6f4b40 x1874672085671296/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828730.376691:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4b40 x1874672085671296/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828730.376696:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828730.376698:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828730.376700:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085671296:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828730.376703:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085671424:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828730.376714:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828730.376719:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828730/real 1787828730] req@ffff9e35da6f6580 x1874672085671424/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828730.376726:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6580 x1874672085671424/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828730.376736:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828730.376738:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828730.376740:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085671424:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828730.376743:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085671552:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828730.376753:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828730.376759:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828730/real 1787828730] req@ffff9e35da6f6940 x1874672085671552/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828730.376765:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6940 x1874672085671552/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828730.376770:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828730.376772:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828730.376774:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085671552:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828730.376778:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085671680:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828730.376788:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828730.376793:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828730/real 1787828730] req@ffff9e35da6f70c0 x1874672085671680/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828730.376852:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f70c0 x1874672085671680/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828730.376858:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828730.376860:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828730.376862:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085671680:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828730.503893:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828730.503934:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828730.503960:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828730.503988:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828731.527919:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787828731.527963:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828731.527964:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828731.527970:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828732.551871:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828732.551878:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828732.551890:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828732.551931:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828733.575892:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828733.575918:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828733.575935:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828733.576054:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828734.599949:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828734.599949:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787828734.599970:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828734.600008:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828735.495944:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828735.495955:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828735.495960:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828735.495970:0:27606:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2170 00000100:00080000:0.0:1787828735.496000:0:27606:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828735.496009:0:27606:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787828735.496083:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7192c0 x1874672085671808/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828735.496140:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828735.496143:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828735.496145:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828735.496148:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2170 00000100:00080000:0.0:1787828735.496151:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828735.496154:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828735.496163:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828735.496176:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718b40 x1874672085671936/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828735.496184:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085671808:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1787828735.496188:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828735.496191:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787828735.496193:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828735.496196:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2170 00000100:00080000:0.0:1787828735.496198:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828735.496200:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828735.496204:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828735.496212:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718f00 x1874672085672064/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828735.496219:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828735.496222:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787828735.496223:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828735.496226:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2170 00000100:00080000:0.0:1787828735.496228:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828735.496230:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828735.496233:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:0.0:1787828735.496243:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a940 x1874672085672192/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828735.496253:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1787828735.496258:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787828735.496261:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787828735.496265:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828735.496272:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2170 00000100:00080000:0.0:1787828735.496278:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828735.496282:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828735.496288:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:2.0:1787828735.496301:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828735/real 1787828735] req@ffff9e35da7192c0 x1874672085671808/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828760 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828735.496304:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a1c0 x1874672085672320/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828735.496340:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828735.496344:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:2.0:1787828735.496348:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7192c0 x1874672085671808/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828760 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828735.496349:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828735.496360:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2170 00000100:00080000:2.0:1787828735.496363:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828735.496365:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828735.496369:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:0.0:1787828735.496374:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787828735.496375:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085671808:192.168.203.141@tcp:250:kworker.0 00000008:00100000:0.0:1787828735.496383:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828735.496399:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7183c0 x1874672085672448/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828735.496403:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085671936:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787828735.496420:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828735.496424:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787828735.496427:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1787828735.496428:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787828735.496437:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2170 00000100:00080000:0.0:1787828735.496443:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787828735.496444:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828735/real 1787828735] req@ffff9e35da718b40 x1874672085671936/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828760 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828735.496453:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787828735.496461:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718b40 x1874672085671936/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828760 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1787828735.496463:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1787828735.496478:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1787828735.496482:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a580 x1874672085672576/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828735.496487:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828735.496496:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085671936:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787828735.496500:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828735.496505:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787828735.496507:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787828735.496510:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085672064:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787828735.496513:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2170 00000100:00080000:0.0:1787828735.496519:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828735.496523:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828735.496528:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000800:00000100:2.0:1787828735.496538:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828735.496542:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da719a40 x1874672085672704/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828735.496550:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787828735.496551:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828735/real 1787828735] req@ffff9e35da718f00 x1874672085672064/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828760 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828735.496555:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787828735.496559:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828735.496567:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2170 00000100:00080000:0.0:1787828735.496573:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828735.496576:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787828735.496580:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672085672064/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828760 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1787828735.496582:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:0.0:1787828735.496592:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718780 x1874672085672832/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828735.496597:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828735.496605:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828735.496611:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085672064:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828735.496620:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085672192:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828735.496638:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828735.496646:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828735/real 1787828735] req@ffff9e35da71a940 x1874672085672192/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828760 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828735.496655:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672085672192/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828760 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828735.496661:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828735.496663:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828735.496666:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085672192:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828735.496669:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085672320:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828735.496680:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828735.496686:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828735/real 1787828735] req@ffff9e35da71a1c0 x1874672085672320/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828760 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828735.496694:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a1c0 x1874672085672320/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828760 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828735.496699:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828735.496701:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828735.496703:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085672320:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828735.496717:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085672448:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828735.496728:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828735.496734:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828735/real 1787828735] req@ffff9e35da7183c0 x1874672085672448/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828760 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828735.496741:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672085672448/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828760 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828735.496747:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828735.496748:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828735.496750:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085672448:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828735.496754:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085672576:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828735.496763:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828735.496769:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828735/real 1787828735] req@ffff9e35da71a580 x1874672085672576/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828760 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828735.496781:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672085672576/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828760 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828735.496787:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828735.496788:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828735.496790:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085672576:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828735.496793:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085672704:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828735.496840:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828735.496853:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828735/real 1787828735] req@ffff9e35da719a40 x1874672085672704/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828760 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828735.496862:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672085672704/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828760 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828735.496868:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828735.496871:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828735.496873:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085672704:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828735.496877:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085672832:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828735.496891:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828735.496904:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828735/real 1787828735] req@ffff9e35da718780 x1874672085672832/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828760 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828735.496912:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718780 x1874672085672832/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828760 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828735.496922:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828735.496924:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828735.496926:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085672832:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828735.623908:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787828735.623909:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828735.623909:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828735.623952:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828736.647905:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828736.647946:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828736.648029:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787828736.648095:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828737.671884:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787828737.671897:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828737.671961:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828737.672082:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828738.695948:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828738.695948:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787828738.695992:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828738.696006:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828739.719870:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787828739.719893:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828739.719942:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828739.719946:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828740.615910:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828740.615921:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828740.615925:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828740.615936:0:27606:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2175 00000100:00080000:1.0:1787828740.615947:0:27606:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828740.615953:0:27606:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787828740.615993:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6d00 x1874672085672960/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828740.616021:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828740.616042:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828740.616044:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828740.616048:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2175 00000100:00080000:1.0:1787828740.616052:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828740.616056:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828740.616066:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828740.616082:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4780 x1874672085673088/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828740.616094:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828740.616097:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787828740.616099:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828740.616107:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2175 00000100:00100000:2.0:1787828740.616110:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085672960:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787828740.616112:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828740.616122:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828740.616127:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828740.616142:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5a40 x1874672085673216/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828740.616162:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828740.616166:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828740.616167:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828740.616171:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2175 00000100:00080000:1.0:1787828740.616178:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828740.616182:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828740.616188:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000800:00000100:2.0:1787828740.616191:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828740.616215:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7840 x1874672085673344/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828740.616231:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828740.616237:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828740.616240:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828740.616247:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2175 00000100:00100000:2.0:1787828740.616255:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828740/real 1787828740] req@ffff9e35da6f6d00 x1874672085672960/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828740.616257:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828740.616270:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828740.616284:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:2.0:1787828740.616296:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6d00 x1874672085672960/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828740.616309:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828740.616315:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828740.616321:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4f00 x1874672085673472/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828740.616326:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085672960:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1787828740.616343:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085673088:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787828740.616364:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828740.616368:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828740.616371:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828740.616377:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2175 00000100:00080000:1.0:1787828740.616382:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1787828740.616384:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787828740.616392:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828740.616400:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828740.616404:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828740/real 1787828740] req@ffff9e35da6f4780 x1874672085673088/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828740.616418:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f70c0 x1874672085673600/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828740.616432:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4780 x1874672085673088/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828740.616456:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828740.616459:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828740.616461:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787828740.616467:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828740.616470:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:1.0:1787828740.616477:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2175 00000100:00100000:2.0:1787828740.616478:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085673088:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787828740.616484:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828740.616492:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828740.616496:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828740.616507:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6940 x1874672085673728/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828740.616517:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085673216:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787828740.616521:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828740.616527:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828740.616529:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828740.616535:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2175 00000100:00080000:1.0:1787828740.616540:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828740.616543:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828740.616548:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000800:00000100:2.0:1787828740.616560:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828740.616566:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6580 x1874672085673856/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828740.616578:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787828740.616581:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828740/real 1787828740] req@ffff9e35da6f5a40 x1874672085673216/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828740.616583:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828740.616588:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828740.616599:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2175 00000100:00080000:1.0:1787828740.616607:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787828740.616612:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5a40 x1874672085673216/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828740.616615:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828740.616624:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00080000:2.0:1787828740.616628:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828740.616637:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828740.616640:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4b40 x1874672085673984/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828740.616645:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085673216:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828740.616662:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085673344:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828740.616687:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828740.616697:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828740/real 1787828740] req@ffff9e35da6f7840 x1874672085673344/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828740.616708:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7840 x1874672085673344/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828740.616723:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828740.616726:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828740.616729:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085673344:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828740.616735:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085673472:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828740.616749:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828740.616757:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828740/real 1787828740] req@ffff9e35da6f4f00 x1874672085673472/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828740.616768:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4f00 x1874672085673472/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828740.616776:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828740.616778:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828740.616781:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085673472:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828740.616786:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085673600:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828740.616845:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828740.616856:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828740/real 1787828740] req@ffff9e35da6f70c0 x1874672085673600/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828740.616868:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f70c0 x1874672085673600/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828740.616876:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828740.616879:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828740.616883:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085673600:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828740.616905:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085673728:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828740.616923:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828740.616932:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828740/real 1787828740] req@ffff9e35da6f6940 x1874672085673728/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828740.616943:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6940 x1874672085673728/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828740.616950:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828740.616953:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828740.616955:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085673728:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828740.616961:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085673856:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828740.616974:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828740.616983:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828740/real 1787828740] req@ffff9e35da6f6580 x1874672085673856/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828740.616993:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6580 x1874672085673856/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828740.617001:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828740.617003:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828740.617006:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085673856:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828740.617011:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085673984:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828740.617048:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828740.617062:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828740/real 1787828740] req@ffff9e35da6f4b40 x1874672085673984/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828740.617074:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4b40 x1874672085673984/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828740.617082:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828740.617085:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828740.617088:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085673984:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828740.743968:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787828740.744019:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787828740.744020:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828740.744044:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828741.767926:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787828741.767933:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787828741.767936:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787828741.768060:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828742.791962:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828742.791969:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787828742.791981:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:0.0:1787828742.792075:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828743.816026:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828743.816035:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828743.816037:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828743.816120:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828744.839948:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787828744.840027:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828744.840045:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787828744.840123:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828745.736109:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828745.736121:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828745.736129:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828745.736145:0:27606:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2181 00000100:00080000:1.0:1787828745.736164:0:27606:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828745.736177:0:27606:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787828745.736243:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f43c0 x1874672085674112/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828745.736314:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828745.736324:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828745.736329:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828745.736338:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2181 00000100:00080000:1.0:1787828745.736348:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828745.736367:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828745.736385:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828745.736408:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4000 x1874672085674240/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828745.736429:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828745.736435:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787828745.736438:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828745.736443:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2181 00000100:00080000:1.0:1787828745.736447:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828745.736452:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828745.736458:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828745.736472:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085674112:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1787828745.736474:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7480 x1874672085674368/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828745.736500:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828745.736504:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828745.736506:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828745.736512:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2181 00000100:00080000:1.0:1787828745.736516:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828745.736521:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828745.736527:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:1.0:1787828745.736542:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5e00 x1874672085674496/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828745.736555:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828745.736559:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828745.736562:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828745.736567:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2181 00000100:00080000:1.0:1787828745.736591:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828745.736596:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828745.736601:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000800:00000100:0.0:1787828745.736615:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828745.736621:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f61c0 x1874672085674624/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828745.736636:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828745.736642:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828745.736645:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828745.736651:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2181 00000100:00080000:1.0:1787828745.736668:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828745.736674:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828745.736680:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828745.736698:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4b40 x1874672085674752/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828745.736712:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828745.736716:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787828745.736719:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828745.736724:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2181 00000100:00080000:1.0:1787828745.736729:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787828745.736736:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828745/real 1787828745] req@ffff9e35da6f43c0 x1874672085674112/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828770 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828745.736737:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828745.736747:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828745.736781:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6580 x1874672085674880/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828745.736849:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787828745.736856:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f43c0 x1874672085674112/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828770 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828745.736861:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828745.736883:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828745.736898:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828745.736898:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2181 00000100:00080000:1.0:1787828745.736912:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828745.736913:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:1.0:1787828745.736930:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787828745.736932:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085674112:192.168.203.141@tcp:250:kworker.0 00000008:00100000:1.0:1787828745.736948:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:0.0:1787828745.736958:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085674240:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828745.736982:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6940 x1874672085675008/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828745.737027:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828745.737044:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828745.737047:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828745.737058:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2181 00000800:00000100:0.0:1787828745.737068:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787828745.737071:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828745.737083:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828745.737091:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:0.0:1787828745.737107:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828745/real 1787828745] req@ffff9e35da6f4000 x1874672085674240/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828770 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828745.737125:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f70c0 x1874672085675136/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787828745.737147:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4000 x1874672085674240/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828770 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828745.737165:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828745.737172:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828745.737178:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085674240:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787828745.737225:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085674368:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787828745.737254:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828745.737269:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828745/real 1787828745] req@ffff9e35da6f7480 x1874672085674368/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828770 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828745.737284:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7480 x1874672085674368/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828770 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828745.737295:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828745.737299:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828745.737303:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085674368:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787828745.737311:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085674496:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787828745.737331:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828745.737343:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828745/real 1787828745] req@ffff9e35da6f5e00 x1874672085674496/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828770 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828745.737357:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5e00 x1874672085674496/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828770 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828745.737376:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828745.737379:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828745.737383:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085674496:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787828745.737390:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085674624:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787828745.737408:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828745.737419:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828745/real 1787828745] req@ffff9e35da6f61c0 x1874672085674624/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828770 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828745.737432:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f61c0 x1874672085674624/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828770 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828745.737443:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828745.737447:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828745.737450:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085674624:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787828745.737457:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085674752:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787828745.737475:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828745.737487:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828745/real 1787828745] req@ffff9e35da6f4b40 x1874672085674752/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828770 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828745.737502:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4b40 x1874672085674752/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828770 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828745.737512:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828745.737516:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828745.737521:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085674752:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787828745.737533:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085674880:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787828745.737569:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828745.737591:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828745/real 1787828745] req@ffff9e35da6f6580 x1874672085674880/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828770 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828745.737618:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6580 x1874672085674880/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828770 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828745.737636:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828745.737642:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828745.737650:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085674880:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787828745.737659:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085675008:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787828745.737697:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828745.737720:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828745/real 1787828745] req@ffff9e35da6f6940 x1874672085675008/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828770 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828745.737746:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6940 x1874672085675008/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828770 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828745.737787:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828745.737792:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828745.737830:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085675008:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787828745.737842:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085675136:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787828745.737887:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828745.737904:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828745/real 1787828745] req@ffff9e35da6f70c0 x1874672085675136/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828770 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828745.737923:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f70c0 x1874672085675136/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828770 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828745.737938:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828745.737945:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828745.737950:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085675136:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828745.864040:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828745.864045:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787828745.864088:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:0.0:1787828745.864232:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828746.887957:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828746.888017:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787828746.888017:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828746.888155:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828747.911911:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828747.911938:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787828747.911950:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787828747.912075:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828748.935932:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828748.935933:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828748.935995:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:0.0:1787828748.936003:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828749.959932:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787828749.959938:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828749.960049:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828749.960088:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828750.856032:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828750.856042:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828750.856047:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828750.856069:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2186 00000100:00080000:0.0:1787828750.856083:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828750.856091:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787828750.856171:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718000 x1874672085675264/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828750.856214:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828750.856217:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828750.856219:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828750.856222:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2186 00000100:00080000:0.0:1787828750.856224:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828750.856227:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828750.856238:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828750.856251:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718780 x1874672085675392/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828750.856260:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828750.856263:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787828750.856266:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828750.856268:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2186 00000100:00080000:0.0:1787828750.856270:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828750.856273:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828750.856276:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828750.856283:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da719a40 x1874672085675520/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828750.856292:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828750.856295:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787828750.856297:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828750.856304:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2186 00000100:00100000:2.0:1787828750.856307:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085675264:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1787828750.856308:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828750.856319:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828750.856328:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:0.0:1787828750.856352:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a580 x1874672085675648/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828750.856366:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828750.856370:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787828750.856373:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828750.856379:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2186 00000100:00080000:0.0:1787828750.856384:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828750.856390:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828750.856397:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000800:00000100:2.0:1787828750.856410:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828750.856416:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7183c0 x1874672085675776/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828750.856424:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828750.856428:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828750.856430:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828750.856435:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2186 00000100:00080000:0.0:1787828750.856438:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828750.856441:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828750.856447:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828750.856476:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a1c0 x1874672085675904/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828750.856483:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828750.856485:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787828750.856486:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828750.856497:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2186 00000100:00080000:0.0:1787828750.856503:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787828750.856504:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828750/real 1787828750] req@ffff9e35da718000 x1874672085675264/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828775 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828750.856515:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828750.856528:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828750.856548:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a940 x1874672085676032/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828750.856566:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787828750.856568:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718000 x1874672085675264/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828775 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828750.856571:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787828750.856579:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828750.856605:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2186 00000100:00080000:0.0:1787828750.856618:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828750.856627:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:2.0:1787828750.856628:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:0.0:1787828750.856633:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00080000:2.0:1787828750.856637:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828750.856647:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085675264:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1787828750.856651:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718f00 x1874672085676160/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828750.856676:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828750.856679:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787828750.856683:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787828750.856683:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085675392:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787828750.856699:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2186 00000100:00080000:0.0:1787828750.856703:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828750.856708:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828750.856711:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:0.0:1787828750.856733:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718b40 x1874672085676288/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787828750.856745:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828750.856768:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828750/real 1787828750] req@ffff9e35da718780 x1874672085675392/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828775 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828750.856783:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718780 x1874672085675392/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828775 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828750.856808:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828750.856811:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828750.856814:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085675392:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828750.856881:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085675520:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828750.856901:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828750.856910:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828750/real 1787828750] req@ffff9e35da719a40 x1874672085675520/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828775 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828750.856920:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672085675520/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828775 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828750.856927:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828750.856930:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828750.856932:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085675520:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828750.856937:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085675648:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828750.856950:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828750.856994:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828750/real 1787828750] req@ffff9e35da71a580 x1874672085675648/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828775 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828750.857015:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672085675648/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828775 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828750.857022:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828750.857024:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828750.857027:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085675648:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828750.857031:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085675776:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828750.857046:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828750.857071:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828750/real 1787828750] req@ffff9e35da7183c0 x1874672085675776/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828775 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828750.857080:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672085675776/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828775 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828750.857087:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828750.857090:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828750.857092:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085675776:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828750.857097:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085675904:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828750.857112:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828750.857130:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828750/real 1787828750] req@ffff9e35da71a1c0 x1874672085675904/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828775 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828750.857152:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a1c0 x1874672085675904/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828775 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828750.857163:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828750.857167:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828750.857172:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085675904:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828750.857180:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085676032:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828750.857201:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828750.857210:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828750/real 1787828750] req@ffff9e35da71a940 x1874672085676032/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828775 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828750.857219:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672085676032/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828775 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828750.857225:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828750.857228:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828750.857230:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085676032:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828750.857235:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085676160:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828750.857247:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828750.857254:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828750/real 1787828750] req@ffff9e35da718f00 x1874672085676160/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828775 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828750.857262:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672085676160/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828775 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828750.857274:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828750.857277:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828750.857279:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085676160:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828750.857284:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085676288:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828750.857296:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828750.857303:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828750/real 1787828750] req@ffff9e35da718b40 x1874672085676288/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828775 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828750.857311:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718b40 x1874672085676288/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828775 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828750.857318:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828750.857320:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828750.857322:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085676288:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787828750.984001:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828750.984001:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828750.984054:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787828750.984073:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828752.007971:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828752.007971:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787828752.007972:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:0.0:1787828752.008076:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828753.032001:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787828753.032018:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828753.032069:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828753.032090:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828754.055960:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828754.056047:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787828754.056053:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:0.0:1787828754.056237:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828755.079898:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787828755.079900:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:3.0:1787828755.079907:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:0.0:1787828755.080033:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828755.976004:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828755.976016:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828755.976021:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828755.976033:0:27606:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2191 00000100:00080000:1.0:1787828755.976045:0:27606:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828755.976053:0:27606:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787828755.976095:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4f00 x1874672085676416/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828755.976130:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828755.976134:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828755.976136:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828755.976141:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2191 00000100:00080000:1.0:1787828755.976145:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828755.976150:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828755.976161:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828755.976176:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7840 x1874672085676544/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828755.976189:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828755.976195:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:2.0:1787828755.976195:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085676416:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787828755.976201:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828755.976213:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2191 00000100:00080000:1.0:1787828755.976217:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828755.976221:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828755.976227:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828755.976241:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5a40 x1874672085676672/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828755.976252:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828755.976255:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828755.976257:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828755.976261:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2191 00000100:00080000:1.0:1787828755.976266:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828755.976269:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828755.976274:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:1.0:1787828755.976292:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4780 x1874672085676800/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787828755.976302:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787828755.976332:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828755.976339:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828755.976343:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828755.976353:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2191 00000100:00100000:2.0:1787828755.976361:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828755/real 1787828755] req@ffff9e35da6f4f00 x1874672085676416/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828780 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828755.976364:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828755.976380:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828755.976393:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:2.0:1787828755.976399:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4f00 x1874672085676416/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828780 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828755.976434:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1787828755.976442:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6d00 x1874672085676928/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828755.976446:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828755.976459:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085676416:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787828755.976484:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828755.976489:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828755.976492:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828755.976499:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2191 00000100:00080000:1.0:1787828755.976506:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787828755.976510:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085676544:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787828755.976517:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828755.976529:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828755.976548:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4f00 x1874672085677056/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828755.976564:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1787828755.976574:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787828755.976575:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787828755.976613:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828755.976624:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2191 00000100:00100000:2.0:1787828755.976627:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828755/real 1787828755] req@ffff9e35da6f7840 x1874672085676544/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828780 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828755.976630:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828755.976640:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828755.976650:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828755.976672:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f70c0 x1874672085677184/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828755.976679:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7840 x1874672085676544/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828780 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828755.976713:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828755.976725:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828755.976730:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828755.976741:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2191 00000100:00080000:2.0:1787828755.976745:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828755.976747:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828755.976759:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828755.976766:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00080000:2.0:1787828755.976768:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828755.976786:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085676544:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828755.976787:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6940 x1874672085677312/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828755.976820:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085676672:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787828755.976863:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828755.976869:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828755.976871:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828755.976878:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2191 00000100:00080000:1.0:1787828755.976883:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828755.976889:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828755.976896:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:1.0:1787828755.976961:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6580 x1874672085677440/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787828755.976983:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828755.977017:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828755/real 1787828755] req@ffff9e35da6f5a40 x1874672085676672/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828780 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828755.977043:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5a40 x1874672085676672/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828780 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828755.977059:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828755.977063:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828755.977067:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085676672:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828755.977076:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085676800:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828755.977105:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828755.977119:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828755/real 1787828755] req@ffff9e35da6f4780 x1874672085676800/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828780 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828755.977132:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4780 x1874672085676800/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828780 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828755.977141:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828755.977145:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828755.977148:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085676800:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828755.977154:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085676928:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828755.977171:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828755.977180:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828755/real 1787828755] req@ffff9e35da6f6d00 x1874672085676928/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828780 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828755.977192:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6d00 x1874672085676928/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828780 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828755.977202:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828755.977205:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828755.977208:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085676928:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828755.977235:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085677056:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828755.977253:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828755.977275:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828755/real 1787828755] req@ffff9e35da6f4f00 x1874672085677056/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828780 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828755.977289:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4f00 x1874672085677056/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828780 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828755.977298:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828755.977301:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828755.977305:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085677056:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828755.977310:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085677184:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828755.977326:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828755.977336:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828755/real 1787828755] req@ffff9e35da6f70c0 x1874672085677184/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828780 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828755.977348:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f70c0 x1874672085677184/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828780 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828755.977356:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828755.977359:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828755.977362:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085677184:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828755.977368:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085677312:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828755.977383:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828755.977392:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828755/real 1787828755] req@ffff9e35da6f6940 x1874672085677312/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828780 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828755.977422:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6940 x1874672085677312/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828780 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828755.977441:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828755.977444:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828755.977447:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085677312:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828755.977453:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085677440:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828755.977479:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828755.977490:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828755/real 1787828755] req@ffff9e35da6f6580 x1874672085677440/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828780 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828755.977503:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6580 x1874672085677440/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828780 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828755.977512:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828755.977515:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828755.977518:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085677440:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787828756.103936:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828756.103944:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787828756.103950:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828756.104040:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828757.127891:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828757.127891:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787828757.127893:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:0.0:1787828757.128032:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828758.151871:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828758.151871:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828758.151943:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:0.0:1787828758.152032:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828759.175854:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828759.175910:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828759.175916:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828759.175932:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828760.199936:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828760.199977:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787828760.199988:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787828760.200009:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828761.095957:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828761.095966:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828761.095970:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828761.095978:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2196 00000100:00080000:0.0:1787828761.095987:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828761.095994:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787828761.096045:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7192c0 x1874672085677568/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828761.096067:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828761.096070:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828761.096071:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828761.096074:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2196 00000100:00080000:0.0:1787828761.096076:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828761.096079:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828761.096087:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828761.096097:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718b40 x1874672085677696/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828761.096105:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085677568:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1787828761.096108:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828761.096115:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787828761.096116:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828761.096120:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2196 00000100:00080000:0.0:1787828761.096122:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828761.096124:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828761.096128:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828761.096136:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718f00 x1874672085677824/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828761.096143:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828761.096145:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787828761.096147:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828761.096149:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2196 00000100:00080000:0.0:1787828761.096153:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828761.096155:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828761.096158:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:0.0:1787828761.096168:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a940 x1874672085677952/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787828761.096168:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787828761.096178:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828761.096181:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787828761.096182:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828761.096185:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2196 00000100:00080000:0.0:1787828761.096188:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828761.096190:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828761.096193:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:0.0:1787828761.096200:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a1c0 x1874672085678080/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828761.096207:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787828761.096209:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828761/real 1787828761] req@ffff9e35da7192c0 x1874672085677568/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828786 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828761.096212:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828761.096215:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828761.096224:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2196 00000100:00080000:0.0:1787828761.096228:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828761.096234:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828761.096241:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828761.096251:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7192c0 x1874672085677568/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828786 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828761.096253:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7183c0 x1874672085678208/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828761.096277:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828761.096279:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787828761.096281:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828761.096285:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2196 00000100:00080000:0.0:1787828761.096289:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828761.096290:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828761.096291:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828761.096300:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1787828761.096303:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828761.096308:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a580 x1874672085678336/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828761.096313:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085677568:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1787828761.096337:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828761.096340:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787828761.096343:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787828761.096347:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085677696:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787828761.096351:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2196 00000100:00080000:0.0:1787828761.096359:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828761.096364:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828761.096369:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000800:00000100:2.0:1787828761.096375:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828761.096389:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da719a40 x1874672085678464/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828761.096390:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828761/real 1787828761] req@ffff9e35da718b40 x1874672085677696/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828786 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828761.096427:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787828761.096432:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718b40 x1874672085677696/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828786 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828761.096433:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787828761.096437:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828761.096446:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2196 00000100:00080000:2.0:1787828761.096453:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828761.096454:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828761.096463:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:0.0:1787828761.096465:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787828761.096469:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085677696:192.168.203.141@tcp:38:kworker.0 00000008:00100000:0.0:1787828761.096475:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:2.0:1787828761.096477:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085677824:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787828761.096493:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718780 x1874672085678592/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787828761.096503:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828761.096514:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828761/real 1787828761] req@ffff9e35da718f00 x1874672085677824/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828786 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828761.096524:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672085677824/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828786 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828761.096534:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828761.096537:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828761.096539:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085677824:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828761.096543:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085677952:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828761.096554:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828761.096560:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828761/real 1787828761] req@ffff9e35da71a940 x1874672085677952/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828786 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828761.096567:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672085677952/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828786 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828761.096573:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828761.096575:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828761.096577:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085677952:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828761.096580:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085678080:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828761.096590:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828761.096597:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828761/real 1787828761] req@ffff9e35da71a1c0 x1874672085678080/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828786 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828761.096609:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a1c0 x1874672085678080/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828786 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828761.096617:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828761.096621:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828761.096625:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085678080:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828761.096632:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085678208:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828761.096653:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828761.096661:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828761/real 1787828761] req@ffff9e35da7183c0 x1874672085678208/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828786 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828761.096670:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672085678208/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828786 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828761.096676:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828761.096678:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828761.096680:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085678208:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828761.096684:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085678336:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828761.096696:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828761.096702:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828761/real 1787828761] req@ffff9e35da71a580 x1874672085678336/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828786 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828761.096710:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672085678336/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828786 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828761.096714:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828761.096716:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828761.096718:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085678336:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828761.096735:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085678464:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828761.096746:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828761.096753:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828761/real 1787828761] req@ffff9e35da719a40 x1874672085678464/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828786 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828761.096760:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672085678464/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828786 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828761.096766:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828761.096768:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828761.096770:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085678464:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828761.096773:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085678592:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828761.096782:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828761.096788:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828761/real 1787828761] req@ffff9e35da718780 x1874672085678592/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828786 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828761.096795:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718780 x1874672085678592/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828786 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828761.096822:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828761.096825:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828761.096827:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085678592:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828761.223877:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787828761.223899:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787828761.223916:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787828761.223960:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828762.247981:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787828762.247989:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787828762.248037:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828762.248044:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828763.271904:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787828763.271960:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828763.271966:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787828763.272044:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828764.295951:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828764.295982:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828764.296033:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828764.296149:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787828765.319962:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828765.319962:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828765.320055:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:0.0:1787828765.320061:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828766.215906:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828766.215916:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828766.215921:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828766.215931:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2201 00000100:00080000:1.0:1787828766.215942:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828766.215949:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787828766.215982:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4b40 x1874672085678720/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828766.216004:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828766.216007:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828766.216008:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828766.216011:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2201 00000100:00080000:1.0:1787828766.216014:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828766.216016:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828766.216024:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828766.216032:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f61c0 x1874672085678848/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828766.216040:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828766.216042:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787828766.216044:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828766.216047:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2201 00000100:00080000:1.0:1787828766.216049:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828766.216052:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828766.216054:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828766.216064:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5e00 x1874672085678976/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828766.216070:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828766.216073:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828766.216074:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828766.216077:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2201 00000100:00080000:1.0:1787828766.216079:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828766.216081:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828766.216084:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:1.0:1787828766.216091:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7480 x1874672085679104/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828766.216098:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828766.216100:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828766.216102:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828766.216111:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2201 00000100:00100000:2.0:1787828766.216111:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085678720:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787828766.216121:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828766.216130:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828766.216133:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:1.0:1787828766.216141:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4000 x1874672085679232/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828766.216147:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828766.216149:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828766.216151:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828766.216154:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2201 00000100:00080000:1.0:1787828766.216157:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828766.216159:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828766.216162:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828766.216179:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f43c0 x1874672085679360/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828766.216184:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828766.216187:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787828766.216189:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828766.216193:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2201 00000100:00080000:1.0:1787828766.216195:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828766.216197:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828766.216200:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828766.216207:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6580 x1874672085679488/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828766.216216:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828766.216218:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828766.216219:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828766.216223:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2201 00000100:00080000:1.0:1787828766.216226:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1787828766.216238:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787828766.216243:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828766.216248:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:1.0:1787828766.216258:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6940 x1874672085679616/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828766.216266:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828766.216270:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828766.216271:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828766.216273:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2201 00000100:00080000:1.0:1787828766.216276:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828766.216278:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828766.216282:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:1.0:1787828766.216288:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f70c0 x1874672085679744/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828766.216299:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828766/real 1787828766] req@ffff9e35da6f4b40 x1874672085678720/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828791 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828766.216320:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4b40 x1874672085678720/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828791 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828766.216330:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828766.216334:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828766.216338:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085678720:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1787828766.216346:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085678848:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828766.216379:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828766.216413:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828766/real 1787828766] req@ffff9e35da6f61c0 x1874672085678848/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828791 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828766.216434:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f61c0 x1874672085678848/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828791 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828766.216450:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828766.216455:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828766.216459:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085678848:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828766.216467:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085678976:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828766.216492:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828766.216504:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828766/real 1787828766] req@ffff9e35da6f5e00 x1874672085678976/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828791 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828766.216523:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5e00 x1874672085678976/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828791 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828766.216532:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828766.216535:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828766.216549:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085678976:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828766.216593:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085679104:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828766.216612:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828766.216623:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828766/real 1787828766] req@ffff9e35da6f7480 x1874672085679104/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828791 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828766.216635:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7480 x1874672085679104/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828791 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828766.216643:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828766.216646:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828766.216649:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085679104:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828766.216655:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085679232:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828766.216681:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828766.216689:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828766/real 1787828766] req@ffff9e35da6f4000 x1874672085679232/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828791 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828766.216699:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4000 x1874672085679232/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828791 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828766.216714:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828766.216717:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828766.216720:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085679232:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828766.216726:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085679360:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828766.216741:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828766.216750:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828766/real 1787828766] req@ffff9e35da6f43c0 x1874672085679360/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828791 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828766.216761:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f43c0 x1874672085679360/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828791 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828766.216769:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828766.216771:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828766.216774:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085679360:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828766.216779:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085679488:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828766.216795:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828766.216889:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828766/real 1787828766] req@ffff9e35da6f6580 x1874672085679488/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828791 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828766.216925:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6580 x1874672085679488/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828791 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828766.216931:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828766.216934:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828766.216937:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085679488:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828766.216943:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085679616:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828766.216961:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828766.216969:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828766/real 1787828766] req@ffff9e35da6f6940 x1874672085679616/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828791 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828766.216977:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6940 x1874672085679616/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828791 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828766.216984:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828766.216986:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828766.216988:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085679616:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828766.216992:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085679744:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828766.217006:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828766.217019:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828766/real 1787828766] req@ffff9e35da6f70c0 x1874672085679744/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828791 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828766.217033:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f70c0 x1874672085679744/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828791 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828766.217044:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828766.217048:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828766.217052:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085679744:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787828766.343956:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828766.343957:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828766.344007:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828766.344042:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828767.367947:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828767.367947:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828767.367951:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787828767.368026:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828768.392012:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828768.392078:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828768.393036:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787828768.393220:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828769.415942:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787828769.415960:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828769.415971:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828769.417141:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828770.439949:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828770.439950:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828770.439985:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:0.0:1787828770.440015:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828771.335952:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828771.335963:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828771.335968:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828771.335978:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2206 00000100:00080000:0.0:1787828771.335991:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828771.335997:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787828771.336050:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718000 x1874672085679872/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828771.337185:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828771.337189:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828771.337191:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828771.337196:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2206 00000100:00080000:0.0:1787828771.337199:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828771.337203:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828771.337225:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828771.337240:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718780 x1874672085680000/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828771.337250:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787828771.337252:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085679872:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1787828771.337256:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787828771.337260:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828771.337267:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2206 00000100:00080000:0.0:1787828771.337272:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828771.337276:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828771.337281:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828771.337294:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da719a40 x1874672085680128/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828771.337304:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828771.337308:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787828771.337310:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828771.337314:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2206 00000100:00080000:0.0:1787828771.337318:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828771.337322:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828771.337327:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:0.0:1787828771.337338:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a580 x1874672085680256/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828771.337349:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828771.337352:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787828771.337354:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828771.337357:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2206 00000100:00080000:0.0:1787828771.337360:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828771.337363:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828771.337367:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:0.0:1787828771.337377:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7183c0 x1874672085680384/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828771.337384:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828771.337388:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828771.337389:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828771.337392:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2206 00000100:00080000:0.0:1787828771.337394:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828771.337398:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828771.337402:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828771.337410:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a1c0 x1874672085680512/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828771.337425:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1787828771.337425:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787828771.337438:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787828771.337440:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828771.337445:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2206 00000100:00080000:0.0:1787828771.337450:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828771.337453:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828771.337460:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828771.337468:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a940 x1874672085680640/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828771.337475:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828771.337478:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787828771.337480:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828771.337483:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2206 00000100:00080000:0.0:1787828771.337487:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828771.337489:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828771.337492:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:0.0:1787828771.337504:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718f00 x1874672085680768/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828771.337506:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828771/real 1787828771] req@ffff9e35da718000 x1874672085679872/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828796 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828771.337519:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828771.337530:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787828771.337533:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828771.337543:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2206 00000100:00080000:0.0:1787828771.337548:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828771.337555:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828771.337559:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:0.0:1787828771.337568:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718b40 x1874672085680896/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828771.337577:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718000 x1874672085679872/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828796 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828771.337595:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828771.337601:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828771.337609:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085679872:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1787828771.337659:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085680000:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828771.337685:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828771.337696:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828771/real 1787828771] req@ffff9e35da718780 x1874672085680000/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828796 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828771.337707:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718780 x1874672085680000/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828796 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828771.337714:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828771.337717:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828771.337720:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085680000:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828771.337726:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085680128:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828771.337738:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828771.337745:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828771/real 1787828771] req@ffff9e35da719a40 x1874672085680128/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828796 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828771.337752:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672085680128/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828796 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828771.337759:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828771.337763:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828771.337766:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085680128:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828771.337773:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085680256:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828771.337794:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828771.337857:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828771/real 1787828771] req@ffff9e35da71a580 x1874672085680256/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828796 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828771.337866:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672085680256/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828796 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828771.337872:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828771.337875:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828771.337877:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085680256:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828771.337882:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085680384:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828771.337897:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828771.337904:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828771/real 1787828771] req@ffff9e35da7183c0 x1874672085680384/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828796 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828771.337912:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672085680384/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828796 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828771.337918:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828771.337920:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828771.337922:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085680384:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828771.337926:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085680512:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828771.337936:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828771.337942:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828771/real 1787828771] req@ffff9e35da71a1c0 x1874672085680512/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828796 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828771.337949:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a1c0 x1874672085680512/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828796 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828771.337954:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828771.337956:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828771.337958:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085680512:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828771.337962:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085680640:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828771.337971:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828771.337977:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828771/real 1787828771] req@ffff9e35da71a940 x1874672085680640/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828796 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828771.337984:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672085680640/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828796 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828771.337989:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828771.337991:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828771.337994:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085680640:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828771.337997:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085680768:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828771.338007:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828771.338013:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828771/real 1787828771] req@ffff9e35da718f00 x1874672085680768/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828796 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828771.338019:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672085680768/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828796 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828771.338025:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828771.338026:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828771.338028:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085680768:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828771.338032:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085680896:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828771.338041:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828771.338047:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828771/real 1787828771] req@ffff9e35da718b40 x1874672085680896/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828796 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828771.338054:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718b40 x1874672085680896/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828796 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828771.338059:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828771.338061:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828771.338064:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085680896:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828771.463908:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787828771.463941:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787828771.463949:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828771.463985:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828772.487994:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787828772.488011:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828772.488079:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828772.488154:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787828773.511976:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828773.511976:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:3.0:1787828773.511976:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:0.0:1787828773.512080:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828774.535935:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828774.535959:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828774.535964:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828774.535976:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787828775.559933:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828775.559979:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828775.559985:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828775.560023:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828776.455887:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828776.455899:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828776.455905:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828776.455917:0:27606:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2211 00000100:00080000:0.0:1787828776.455929:0:27606:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828776.455937:0:27606:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787828776.455978:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7192c0 x1874672085681024/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828776.456005:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828776.456008:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828776.456010:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828776.456013:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2211 00000100:00080000:0.0:1787828776.456016:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828776.456018:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828776.456028:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828776.456039:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718b40 x1874672085681152/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828776.456047:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828776.456051:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787828776.456053:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828776.456056:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2211 00000100:00080000:0.0:1787828776.456058:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828776.456060:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828776.456063:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828776.456071:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718f00 x1874672085681280/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828776.456079:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828776.456083:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:2.0:1787828776.456088:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085681024:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1787828776.456092:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828776.456104:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2211 00000100:00080000:0.0:1787828776.456106:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828776.456109:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828776.456113:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:0.0:1787828776.456124:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a940 x1874672085681408/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828776.456134:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828776.456137:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787828776.456138:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828776.456143:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2211 00000100:00080000:0.0:1787828776.456146:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828776.456148:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828776.456153:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000800:00000100:2.0:1787828776.456172:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828776.456173:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a1c0 x1874672085681536/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828776.456184:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828776.456188:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828776.456189:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828776.456193:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2211 00000100:00080000:0.0:1787828776.456197:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828776.456200:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828776.456205:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828776.456215:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7183c0 x1874672085681664/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828776.456221:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828776.456225:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787828776.456226:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828776.456233:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2211 00000100:00100000:2.0:1787828776.456233:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828776/real 1787828776] req@ffff9e35da7192c0 x1874672085681024/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828801 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828776.456244:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828776.456256:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828776.456268:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828776.456279:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a580 x1874672085681792/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828776.456289:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7192c0 x1874672085681024/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828801 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828776.456291:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828776.456302:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787828776.456305:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828776.456315:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2211 00000100:00080000:0.0:1787828776.456323:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828776.456327:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828776.456330:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00080000:2.0:1787828776.456330:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1787828776.456342:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da719a40 x1874672085681920/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828776.456344:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828776.456361:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085681024:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1787828776.456363:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828776.456372:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787828776.456377:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828776.456382:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2211 00000100:00080000:0.0:1787828776.456386:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787828776.456388:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085681152:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787828776.456391:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828776.456396:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:0.0:1787828776.456405:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718780 x1874672085682048/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787828776.456430:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828776.456452:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828776/real 1787828776] req@ffff9e35da718b40 x1874672085681152/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828801 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828776.456462:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718b40 x1874672085681152/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828801 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828776.456468:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828776.456472:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828776.456476:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085681152:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828776.456509:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085681280:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828776.456531:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828776.456544:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828776/real 1787828776] req@ffff9e35da718f00 x1874672085681280/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828801 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828776.456557:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672085681280/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828801 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828776.456566:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828776.456569:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828776.456573:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085681280:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828776.456578:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085681408:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828776.456594:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828776.456602:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828776/real 1787828776] req@ffff9e35da71a940 x1874672085681408/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828801 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828776.456609:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672085681408/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828801 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828776.456615:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828776.456618:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828776.456620:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085681408:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828776.456623:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085681536:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828776.456634:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828776.456639:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828776/real 1787828776] req@ffff9e35da71a1c0 x1874672085681536/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828801 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828776.456647:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a1c0 x1874672085681536/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828801 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828776.456653:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828776.456655:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828776.456656:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085681536:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828776.456660:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085681664:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828776.456670:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828776.456676:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828776/real 1787828776] req@ffff9e35da7183c0 x1874672085681664/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828801 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828776.456683:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672085681664/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828801 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828776.456688:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828776.456690:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828776.456692:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085681664:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828776.456695:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085681792:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828776.456705:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828776.456710:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828776/real 1787828776] req@ffff9e35da71a580 x1874672085681792/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828801 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828776.456717:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672085681792/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828801 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828776.456727:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828776.456729:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828776.456731:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085681792:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828776.456734:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085681920:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828776.456743:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828776.456749:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828776/real 1787828776] req@ffff9e35da719a40 x1874672085681920/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828801 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828776.456755:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672085681920/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828801 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828776.456760:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828776.456762:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828776.456764:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085681920:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828776.456767:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085682048:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828776.457073:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828776.457096:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828776/real 1787828776] req@ffff9e35da718780 x1874672085682048/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828801 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828776.457124:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718780 x1874672085682048/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828801 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828776.457133:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828776.457137:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828776.457141:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085682048:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828776.583893:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787828776.583925:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828776.583946:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:0.0:1787828776.583998:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828777.607900:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828777.607913:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828777.607933:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828777.607953:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787828778.631885:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828778.631885:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787828778.631913:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787828778.631930:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828779.655948:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787828779.655954:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828779.655985:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787828779.656039:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828780.679885:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787828780.679923:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828780.679957:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828780.679974:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828781.575882:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828781.575890:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828781.575894:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828781.575902:0:27606:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2216 00000100:00080000:1.0:1787828781.575912:0:27606:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828781.575918:0:27606:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787828781.575949:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4f00 x1874672085682176/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828781.575978:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828781.575981:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828781.575983:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828781.575986:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2216 00000100:00080000:1.0:1787828781.575989:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828781.575991:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828781.575999:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828781.576010:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6d00 x1874672085682304/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828781.576019:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787828781.576022:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085682176:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787828781.576023:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787828781.576028:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828781.576036:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2216 00000100:00080000:1.0:1787828781.576040:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828781.576045:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828781.576051:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828781.576069:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4780 x1874672085682432/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828781.576081:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828781.576087:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000100:2.0:1787828781.576087:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787828781.576091:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828781.576098:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2216 00000100:00080000:1.0:1787828781.576102:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828781.576106:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828781.576111:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:1.0:1787828781.576125:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5a40 x1874672085682560/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828781.576129:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828781/real 1787828781] req@ffff9e35da6f4f00 x1874672085682176/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828806 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828781.576144:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828781.576147:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828781.576150:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828781.576157:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2216 00000100:00080000:1.0:1787828781.576161:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828781.576164:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828781.576169:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:2.0:1787828781.576169:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4f00 x1874672085682176/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828806 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828781.576182:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7840 x1874672085682688/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828781.576183:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828781.576193:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:1.0:1787828781.576196:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828781.576201:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:2.0:1787828781.576202:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085682176:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787828781.576205:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828781.576214:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2216 00000100:00080000:1.0:1787828781.576217:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828781.576219:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828781.576222:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828781.576230:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f70c0 x1874672085682816/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828781.576237:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828781.576242:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:2.0:1787828781.576243:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085682304:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787828781.576245:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828781.576254:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2216 00000100:00080000:1.0:1787828781.576259:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828781.576263:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828781.576269:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1787828781.576282:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828781.576287:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6940 x1874672085682944/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828781.576299:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787828781.576302:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828781/real 1787828781] req@ffff9e35da6f6d00 x1874672085682304/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828806 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828781.576304:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828781.576308:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828781.576316:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2216 00000100:00080000:1.0:1787828781.576322:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787828781.576327:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6d00 x1874672085682304/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828806 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828781.576328:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828781.576336:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00080000:2.0:1787828781.576350:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1787828781.576355:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6580 x1874672085683072/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828781.576356:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828781.576363:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085682304:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828781.576375:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085682432:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787828781.576380:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828781.576385:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828781.576388:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828781.576395:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2216 00000800:00000100:2.0:1787828781.576399:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787828781.576402:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828781.576409:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828781.576415:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:2.0:1787828781.576416:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828781/real 1787828781] req@ffff9e35da6f4780 x1874672085682432/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828806 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828781.576431:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f43c0 x1874672085683200/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828781.576432:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4780 x1874672085682432/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828806 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828781.576464:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828781.576467:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828781.576470:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085682432:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828781.576475:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085682560:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828781.576489:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828781.576496:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828781/real 1787828781] req@ffff9e35da6f5a40 x1874672085682560/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828806 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828781.576504:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5a40 x1874672085682560/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828806 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828781.576509:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828781.576511:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828781.576514:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085682560:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828781.576518:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085682688:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828781.576528:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828781.576534:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828781/real 1787828781] req@ffff9e35da6f7840 x1874672085682688/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828806 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828781.576541:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7840 x1874672085682688/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828806 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828781.576546:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828781.576548:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828781.576550:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085682688:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828781.576567:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085682816:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828781.576578:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828781.576584:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828781/real 1787828781] req@ffff9e35da6f70c0 x1874672085682816/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828806 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828781.576592:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f70c0 x1874672085682816/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828806 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828781.576597:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828781.576599:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828781.576601:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085682816:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828781.576604:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085682944:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828781.576614:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828781.576620:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828781/real 1787828781] req@ffff9e35da6f6940 x1874672085682944/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828806 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828781.576626:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6940 x1874672085682944/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828806 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828781.576631:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828781.576633:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828781.576635:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085682944:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828781.576639:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085683072:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828781.576648:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828781.576653:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828781/real 1787828781] req@ffff9e35da6f6580 x1874672085683072/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828806 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828781.576660:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6580 x1874672085683072/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828806 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828781.576665:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828781.576667:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828781.576669:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085683072:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828781.576672:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085683200:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828781.576680:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828781.576686:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828781/real 1787828781] req@ffff9e35da6f43c0 x1874672085683200/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828806 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828781.576692:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f43c0 x1874672085683200/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828806 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828781.576698:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828781.576700:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828781.576702:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085683200:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828781.703913:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787828781.703952:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828781.703976:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828781.703984:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828782.727933:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828782.727933:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787828782.727971:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828782.727977:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828783.751999:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828783.751999:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828783.752000:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828783.752045:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828784.775877:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828784.775920:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828784.775950:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828784.775951:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787828785.799901:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828785.799901:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828785.799927:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828785.799936:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787828786.695945:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828786.695954:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828786.695958:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828786.695969:0:27606:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2222 00000100:00080000:1.0:1787828786.695991:0:27606:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828786.696000:0:27606:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787828786.696040:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4000 x1874672085683328/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828786.696066:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828786.696070:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828786.696072:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828786.696075:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2222 00000100:00080000:1.0:1787828786.696079:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828786.696083:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828786.696092:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828786.696103:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7480 x1874672085683456/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828786.696112:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085683328:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787828786.696135:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828786.696141:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787828786.696144:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828786.696151:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2222 00000100:00080000:1.0:1787828786.696155:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828786.696160:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828786.696167:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828786.696182:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5e00 x1874672085683584/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828786.696190:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828786.696193:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828786.696195:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828786.696198:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2222 00000100:00080000:1.0:1787828786.696202:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828786.696204:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828786.696226:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:1.0:1787828786.696237:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f61c0 x1874672085683712/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828786.696245:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085683456:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787828786.696250:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828786.696253:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828786.696255:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828786.696259:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2222 00000100:00080000:1.0:1787828786.696262:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828786.696266:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828786.696270:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:2.0:1787828786.696278:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085683584:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828786.696285:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4b40 x1874672085683840/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828786.696294:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828786.696298:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828786.696300:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828786.696307:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2222 00000100:00100000:2.0:1787828786.696308:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085683712:192.168.203.141@tcp:8:kworker.0 00000100:00080000:1.0:1787828786.696311:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828786.696324:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828786.696328:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828786.696342:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085683840:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828786.696343:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f43c0 x1874672085683968/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828786.696357:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828786.696360:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787828786.696363:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787828786.696368:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085683968:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787828786.696369:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2222 00000100:00080000:1.0:1787828786.696375:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828786.696380:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828786.696384:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828786.696396:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6580 x1874672085684096/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828786.696411:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828786.696415:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828786.696416:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828786.696420:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2222 00000100:00080000:1.0:1787828786.696423:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828786.696426:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787828786.696431:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085684096:192.168.203.141@tcp:38:kworker.0 00000008:00100000:1.0:1787828786.696434:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:1.0:1787828786.696453:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6940 x1874672085684224/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828786.696472:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828786.696477:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828786.696479:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828786.696485:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2222 00000100:00080000:1.0:1787828786.696491:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787828786.696494:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085684224:192.168.203.141@tcp:8:kworker.0 00000100:00080000:1.0:1787828786.696501:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828786.696508:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:1.0:1787828786.696523:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f70c0 x1874672085684352/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828786.696574:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085684352:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828786.823921:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828786.823941:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787828786.823943:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828786.823993:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828787.847912:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828787.847941:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828787.848009:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828787.848061:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787828788.871882:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828788.871888:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828788.871892:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828788.871941:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1787828789.768207:0:2397:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.141:988 00000400:00000100:0.0:1787828789.768268:0:2397:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.141@tcp at host 192.168.203.141 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1787828789.768290:0:2397:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:0.0:1787828789.768351:0:2397:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000800:00000100:0.0:1787828789.768370:0:2397:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:0.0:1787828789.768388:0:2397:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:0.0:1787828789.768417:0:2397:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:0.0:1787828789.768435:0:2397:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:2.0:1787828789.768445:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828786/real 1787828789] req@ffff9e35da6f70c0 x1874672085684352/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828811 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:0.0:1787828789.768456:0:2397:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:0.0:1787828789.768476:0:2397:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:0.0:1787828789.768494:0:2397:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:2.0:1787828789.768514:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f70c0 x1874672085684352/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828811 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828789.768533:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828789.768539:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828789.768544:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085684352:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828789.768553:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828786/real 1787828789] req@ffff9e35da6f6940 x1874672085684224/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828811 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828789.768569:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6940 x1874672085684224/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828811 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828789.768579:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828789.768582:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828789.768586:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085684224:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828789.768593:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828786/real 1787828789] req@ffff9e35da6f6580 x1874672085684096/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828811 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828789.768605:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6580 x1874672085684096/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828811 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828789.768612:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828789.768616:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828789.768619:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085684096:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828789.768626:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828786/real 1787828789] req@ffff9e35da6f43c0 x1874672085683968/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828811 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828789.768639:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f43c0 x1874672085683968/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828811 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828789.768646:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828789.768650:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828789.768653:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085683968:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828789.768660:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828786/real 1787828789] req@ffff9e35da6f61c0 x1874672085683712/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828811 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828789.768672:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f61c0 x1874672085683712/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828811 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828789.768695:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828789.768699:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828789.768703:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085683712:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828789.768711:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828786/real 1787828789] req@ffff9e35da6f4b40 x1874672085683840/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828811 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828789.768724:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4b40 x1874672085683840/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828811 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828789.768734:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828789.768738:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828789.768741:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085683840:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828789.768750:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828786/real 1787828789] req@ffff9e35da6f7480 x1874672085683456/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828811 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828789.768763:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7480 x1874672085683456/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828811 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828789.768772:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828789.768776:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828789.768779:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085683456:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828789.768786:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828786/real 1787828789] req@ffff9e35da6f5e00 x1874672085683584/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828811 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828789.768841:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5e00 x1874672085683584/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828811 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828789.768851:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828789.768855:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828789.768859:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085683584:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828789.768868:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828786/real 1787828789] req@ffff9e35da6f4000 x1874672085683328/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828811 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828789.768882:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4000 x1874672085683328/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828811 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828789.768890:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828789.768894:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828789.768897:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085683328:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1787828789.895959:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787828789.895964:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828789.895997:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828789.896039:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828790.919891:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787828790.919911:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828790.919938:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828790.919946:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00080000:0.0:1787828791.815960:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828791.815968:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828791.815971:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828791.815979:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2227 00000100:00080000:0.0:1787828791.815989:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828791.815995:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787828791.816024:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718000 x1874672085684480/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828791.816063:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828791.816067:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828791.816069:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828791.816074:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2227 00000100:00080000:0.0:1787828791.816077:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828791.816081:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828791.816092:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828791.816109:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718780 x1874672085684608/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828791.816123:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787828791.816124:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085684480:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1787828791.816140:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787828791.816143:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828791.816148:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2227 00000100:00080000:0.0:1787828791.816153:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828791.816158:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828791.816163:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828791.816179:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da719a40 x1874672085684736/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828791.816190:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828791.816194:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787828791.816196:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1787828791.816202:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787828791.816206:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2227 00000100:00080000:0.0:1787828791.816213:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828791.816219:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828791.816226:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:0.0:1787828791.816240:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a580 x1874672085684864/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828791.816252:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828791/real 1787828791] req@ffff9e35da718000 x1874672085684480/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828816 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828791.816254:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828791.816263:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787828791.816266:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828791.816271:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2227 00000100:00080000:0.0:1787828791.816275:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828791.816281:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828791.816291:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:2.0:1787828791.816308:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718000 x1874672085684480/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828816 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828791.816323:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1787828791.816326:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7183c0 x1874672085684992/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828791.816331:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828791.816342:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085684480:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1787828791.816347:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828791.816352:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828791.816355:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828791.816360:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2227 00000100:00080000:0.0:1787828791.816364:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828791.816369:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828791.816376:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828791.816377:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085684608:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787828791.816392:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a1c0 x1874672085685120/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828791.816410:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1787828791.816412:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787828791.816416:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787828791.816420:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828791.816426:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2227 00000100:00080000:0.0:1787828791.816430:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787828791.816436:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828791/real 1787828791] req@ffff9e35da718780 x1874672085684608/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828816 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828791.816437:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828791.816446:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828791.816461:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718780 x1874672085684608/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828816 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828791.816466:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a940 x1874672085685248/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828791.816478:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828791.816490:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:0.0:1787828791.816494:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787828791.816498:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085684608:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787828791.816501:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787828791.816506:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787828791.816511:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085684736:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787828791.816516:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2227 00000100:00080000:0.0:1787828791.816523:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828791.816528:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828791.816534:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:0.0:1787828791.816548:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718f00 x1874672085685376/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787828791.816558:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787828791.816565:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828791.816570:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787828791.816573:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787828791.816578:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828791/real 1787828791] req@ffff9e35da719a40 x1874672085684736/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828816 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828791.816582:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2227 00000100:00080000:0.0:1787828791.816590:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828791.816598:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787828791.816604:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672085684736/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828816 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1787828791.816607:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00080000:2.0:1787828791.816621:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1787828791.816628:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718b40 x1874672085685504/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828791.816628:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828791.816637:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085684736:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828791.816651:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085684864:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828791.816698:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828791.816712:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828791/real 1787828791] req@ffff9e35da71a580 x1874672085684864/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828816 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828791.816725:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672085684864/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828816 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828791.816734:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828791.816738:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828791.816741:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085684864:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828791.816748:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085684992:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828791.816768:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828791.816779:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828791/real 1787828791] req@ffff9e35da7183c0 x1874672085684992/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828816 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828791.816792:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672085684992/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828816 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828791.816859:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828791.816863:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828791.816866:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085684992:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828791.816889:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085685120:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828791.816913:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828791.816926:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828791/real 1787828791] req@ffff9e35da71a1c0 x1874672085685120/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828816 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828791.816938:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a1c0 x1874672085685120/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828816 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828791.816948:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828791.816953:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828791.816956:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085685120:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828791.816967:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085685248:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828791.816987:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828791.816998:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828791/real 1787828791] req@ffff9e35da71a940 x1874672085685248/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828816 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828791.817010:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672085685248/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828816 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828791.817019:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828791.817023:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828791.817027:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085685248:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828791.817033:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085685376:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828791.817060:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828791.817073:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828791/real 1787828791] req@ffff9e35da718f00 x1874672085685376/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828816 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828791.817086:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672085685376/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828816 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828791.817095:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828791.817099:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828791.817102:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085685376:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828791.817109:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085685504:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828791.817130:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828791.817141:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828791/real 1787828791] req@ffff9e35da718b40 x1874672085685504/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828816 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828791.817154:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718b40 x1874672085685504/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828816 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828791.817163:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828791.817171:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828791.817174:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085685504:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787828791.943899:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828791.943899:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828791.943945:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828791.943963:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828792.967932:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828792.967994:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787828792.968007:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828792.968030:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828793.991907:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828793.991915:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828793.991921:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828793.991931:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828795.015906:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828795.015906:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787828795.015937:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787828795.015957:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828796.039953:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828796.039978:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787828796.039992:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828796.040016:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828796.936008:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828796.936019:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828796.936024:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828796.936050:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2232 00000100:00080000:1.0:1787828796.936062:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828796.936068:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787828796.936129:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7840 x1874672085685632/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828796.936173:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828796.936176:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828796.936177:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828796.936180:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2232 00000100:00080000:1.0:1787828796.936183:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828796.936185:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828796.936195:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828796.936206:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5a40 x1874672085685760/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828796.936213:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828796.936215:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787828796.936217:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828796.936219:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2232 00000100:00080000:1.0:1787828796.936221:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828796.936224:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828796.936228:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828796.936234:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4780 x1874672085685888/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828796.936241:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828796.936243:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828796.936244:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828796.936247:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2232 00000100:00080000:1.0:1787828796.936249:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828796.936251:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828796.936254:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:1.0:1787828796.936263:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6d00 x1874672085686016/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828796.936270:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085685632:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787828796.936273:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828796.936284:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828796.936289:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828796.936294:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2232 00000100:00080000:1.0:1787828796.936296:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828796.936299:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828796.936302:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:1.0:1787828796.936312:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4f00 x1874672085686144/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828796.936318:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828796.936321:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828796.936322:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828796.936325:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2232 00000100:00080000:1.0:1787828796.936327:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828796.936329:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828796.936332:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828796.936340:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4000 x1874672085686272/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828796.936346:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828796.936349:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787828796.936351:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828796.936353:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2232 00000100:00080000:1.0:1787828796.936356:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828796.936358:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828796.936362:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1787828796.936362:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828796.936380:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5e00 x1874672085686400/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828796.936390:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828796.936393:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828796.936395:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828796.936398:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2232 00000100:00080000:1.0:1787828796.936400:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828796.936403:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828796.936408:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:2.0:1787828796.936418:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828796/real 1787828796] req@ffff9e35da6f7840 x1874672085685632/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828821 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828796.936419:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7480 x1874672085686528/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828796.936456:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828796.936460:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828796.936462:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787828796.936465:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7840 x1874672085685632/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828821 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828796.936470:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2232 00000100:00080000:1.0:1787828796.936480:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828796.936489:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:2.0:1787828796.936489:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:1.0:1787828796.936507:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00080000:2.0:1787828796.936508:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828796.936526:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085685632:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1787828796.936542:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4b40 x1874672085686656/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828796.936543:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085685760:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828796.936587:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828796.936601:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828796/real 1787828796] req@ffff9e35da6f5a40 x1874672085685760/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828821 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828796.936611:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5a40 x1874672085685760/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828821 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828796.936618:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828796.936621:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828796.936623:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085685760:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828796.936630:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085685888:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828796.936658:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828796.936674:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828796/real 1787828796] req@ffff9e35da6f4780 x1874672085685888/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828821 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828796.936682:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4780 x1874672085685888/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828821 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828796.936688:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828796.936690:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828796.936692:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085685888:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828796.936718:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085686016:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828796.936729:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828796.936735:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828796/real 1787828796] req@ffff9e35da6f6d00 x1874672085686016/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828821 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828796.936743:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6d00 x1874672085686016/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828821 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828796.936748:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828796.936750:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828796.936752:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085686016:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828796.936756:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085686144:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828796.936765:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828796.936771:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828796/real 1787828796] req@ffff9e35da6f4f00 x1874672085686144/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828821 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828796.936779:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4f00 x1874672085686144/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828821 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828796.936785:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828796.936786:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828796.936788:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085686144:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828796.936792:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085686272:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828796.936840:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828796.936847:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828796/real 1787828796] req@ffff9e35da6f4000 x1874672085686272/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828821 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828796.936855:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4000 x1874672085686272/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828821 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828796.936861:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828796.936863:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828796.936865:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085686272:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828796.936870:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085686400:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828796.936880:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828796.936886:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828796/real 1787828796] req@ffff9e35da6f5e00 x1874672085686400/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828821 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828796.936921:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5e00 x1874672085686400/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828821 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828796.936938:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828796.936942:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828796.936946:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085686400:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828796.936954:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085686528:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828796.936976:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828796.936988:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828796/real 1787828796] req@ffff9e35da6f7480 x1874672085686528/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828821 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828796.937001:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7480 x1874672085686528/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828821 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828796.937011:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828796.937016:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828796.937020:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085686528:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828796.937028:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085686656:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828796.937048:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828796.937058:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828796/real 1787828796] req@ffff9e35da6f4b40 x1874672085686656/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828821 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828796.937070:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4b40 x1874672085686656/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828821 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828796.937087:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828796.937091:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828796.937095:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085686656:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828797.063953:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828797.063960:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828797.064006:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:1.0:1787828797.064023:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828798.087937:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828798.087966:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787828798.087971:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:1.0:1787828798.087989:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828799.111976:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828799.111985:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828799.112074:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828799.112074:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828800.135884:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828800.135891:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:3.0:1787828800.135926:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787828800.136033:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828801.159928:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828801.159928:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:3.0:1787828801.159928:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787828801.160028:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828802.055944:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828802.055959:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828802.055964:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828802.055978:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2237 00000100:00080000:1.0:1787828802.055992:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828802.056002:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787828802.056057:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f61c0 x1874672085686784/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828802.056092:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828802.056095:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828802.056096:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828802.056100:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2237 00000100:00080000:1.0:1787828802.056103:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828802.056107:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828802.056117:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828802.056130:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f43c0 x1874672085686912/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828802.056138:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828802.056141:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787828802.056143:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828802.056148:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2237 00000100:00100000:2.0:1787828802.056149:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085686784:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787828802.056151:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828802.056159:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828802.056163:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828802.056171:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6580 x1874672085687040/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828802.056177:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828802.056180:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828802.056181:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828802.056184:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2237 00000100:00080000:1.0:1787828802.056187:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828802.056190:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828802.056194:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:1.0:1787828802.056202:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6940 x1874672085687168/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828802.056208:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828802.056211:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828802.056213:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828802.056216:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2237 00000100:00080000:1.0:1787828802.056218:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828802.056220:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828802.056223:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000800:00000100:2.0:1787828802.056229:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828802.056234:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f70c0 x1874672085687296/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828802.056242:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828802.056246:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828802.056248:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828802.056252:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2237 00000100:00080000:1.0:1787828802.056254:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828802.056257:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828802.056260:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828802.056269:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4b40 x1874672085687424/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828802.056279:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828802/real 1787828802] req@ffff9e35da6f61c0 x1874672085686784/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828827 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828802.056285:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828802.056295:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787828802.056297:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828802.056302:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2237 00000100:00080000:1.0:1787828802.056305:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787828802.056309:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f61c0 x1874672085686784/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828827 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828802.056312:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828802.056318:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828802.056331:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7480 x1874672085687552/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828802.056333:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828802.056339:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828802.056342:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828802.056344:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828802.056352:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2237 00000100:00080000:2.0:1787828802.056355:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:1.0:1787828802.056356:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787828802.056362:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085686784:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787828802.056364:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828802.056373:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:1.0:1787828802.056381:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5e00 x1874672085687680/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828802.056387:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828802.056389:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828802.056390:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828802.056393:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2237 00000100:00080000:1.0:1787828802.056396:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787828802.056401:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085686912:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787828802.056402:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828802.056410:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:1.0:1787828802.056421:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4000 x1874672085687808/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787828802.056436:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828802.056449:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828802/real 1787828802] req@ffff9e35da6f43c0 x1874672085686912/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828827 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828802.056461:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f43c0 x1874672085686912/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828827 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828802.056473:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828802.056476:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828802.056478:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085686912:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828802.056483:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085687040:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828802.056496:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828802.056502:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828802/real 1787828802] req@ffff9e35da6f6580 x1874672085687040/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828827 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828802.056511:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6580 x1874672085687040/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828827 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828802.056517:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828802.056519:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828802.056521:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085687040:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828802.056525:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085687168:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828802.056536:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828802.056542:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828802/real 1787828802] req@ffff9e35da6f6940 x1874672085687168/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828827 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828802.056550:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6940 x1874672085687168/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828827 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828802.056556:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828802.056558:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828802.056561:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085687168:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828802.056565:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085687296:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828802.056574:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828802.056580:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828802/real 1787828802] req@ffff9e35da6f70c0 x1874672085687296/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828827 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828802.056587:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f70c0 x1874672085687296/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828827 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828802.056593:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828802.056608:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828802.056613:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085687296:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828802.056618:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085687424:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828802.056634:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828802.056659:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828802/real 1787828802] req@ffff9e35da6f4b40 x1874672085687424/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828827 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828802.056668:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4b40 x1874672085687424/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828827 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828802.056674:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828802.056677:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828802.056679:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085687424:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828802.056683:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085687552:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828802.056696:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828802.056708:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828802/real 1787828802] req@ffff9e35da6f7480 x1874672085687552/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828827 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828802.056722:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7480 x1874672085687552/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828827 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828802.056732:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828802.056736:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828802.056740:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085687552:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828802.056748:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085687680:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828802.056769:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828802.056777:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828802/real 1787828802] req@ffff9e35da6f5e00 x1874672085687680/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828827 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828802.056786:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5e00 x1874672085687680/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828827 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828802.056794:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828802.056798:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828802.056802:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085687680:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828802.056871:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085687808:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828802.056897:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828802.056911:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828802/real 1787828802] req@ffff9e35da6f4000 x1874672085687808/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828827 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828802.056924:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4000 x1874672085687808/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828827 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828802.056931:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828802.056933:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828802.056936:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085687808:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828802.183911:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:3.0:1787828802.183911:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787828802.183930:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828802.184013:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828803.207943:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828803.207943:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828803.207972:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787828803.207985:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828804.231891:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828804.231891:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828804.231897:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787828804.231951:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828805.255949:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:3.0:1787828805.255949:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787828805.255950:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828805.256090:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828806.279967:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828806.279979:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828806.280055:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787828806.280187:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828807.175959:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828807.175975:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828807.175981:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828807.175997:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2242 00000100:00080000:2.0:1787828807.176014:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828807.176025:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787828807.176081:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f840 x1874672085687936/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828807.176125:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828807.176131:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828807.176134:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828807.176140:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2242 00000100:00080000:2.0:1787828807.176146:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828807.176151:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828807.176166:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828807.176184:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c000 x1874672085688064/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828807.176213:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828807.176218:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787828807.176222:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787828807.176227:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085687936:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1787828807.176233:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2242 00000100:00080000:2.0:1787828807.176242:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828807.176249:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828807.176256:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828807.176274:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cb40 x1874672085688192/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828807.176287:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828807.176292:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787828807.176295:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828807.176301:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2242 00000100:00080000:2.0:1787828807.176306:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828807.176312:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:0.0:1787828807.176319:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:2.0:1787828807.176320:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:2.0:1787828807.176339:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e1c0 x1874672085688320/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828807.176352:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828807.176367:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787828807.176370:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828807.176377:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2242 00000100:00080000:2.0:1787828807.176383:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787828807.176388:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828807/real 1787828807] req@ffff9e35c498f840 x1874672085687936/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828832 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828807.176394:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828807.176409:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:0.0:1787828807.176419:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f840 x1874672085687936/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828832 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828807.176430:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f480 x1874672085688448/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828807.176447:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828807.176458:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:2.0:1787828807.176461:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787828807.176466:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085687936:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1787828807.176472:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828807.176478:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828807.176486:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2242 00000100:00080000:2.0:1787828807.176492:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828807.176498:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787828807.176503:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085688064:192.168.203.141@tcp:38:kworker.0 00000008:00100000:2.0:1787828807.176509:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828807.176533:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f0c0 x1874672085688576/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787828807.176538:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828807.176555:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828807/real 1787828807] req@ffff9e35c498c000 x1874672085688064/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828832 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828807.176559:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828807.176591:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787828807.176596:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787828807.176598:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c000 x1874672085688064/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828832 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828807.176609:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2242 00000100:00080000:2.0:1787828807.176627:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828807.176628:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828807.176638:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:2.0:1787828807.176640:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787828807.176646:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085688064:192.168.203.141@tcp:38:kworker.0 00000008:00100000:2.0:1787828807.176651:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828807.176657:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085688192:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828807.176671:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e580 x1874672085688704/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787828807.176682:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1787828807.176696:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787828807.176698:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828807/real 1787828807] req@ffff9e35c498cb40 x1874672085688192/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828832 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828807.176708:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787828807.176715:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787828807.176720:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498cb40 x1874672085688192/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828832 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828807.176729:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2242 00000100:00080000:0.0:1787828807.176734:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828807.176740:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828807.176742:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828807.176750:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085688192:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1787828807.176756:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787828807.176762:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085688320:192.168.203.141@tcp:8:kworker.0 00000008:00100000:2.0:1787828807.176770:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000800:00000100:0.0:1787828807.176785:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828807.176796:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cf00 x1874672085688832/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787828807.176831:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828807/real 1787828807] req@ffff9e35c498e1c0 x1874672085688320/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828832 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828807.176879:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787828807.176883:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e1c0 x1874672085688320/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828832 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828807.176886:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787828807.176891:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828807.176904:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2242 00000100:00080000:0.0:1787828807.176906:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828807.176910:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828807.176913:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:2.0:1787828807.176922:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787828807.176923:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085688320:192.168.203.141@tcp:8:kworker.0 00000008:00100000:2.0:1787828807.176936:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:0.0:1787828807.176937:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085688448:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828807.176959:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498d680 x1874672085688960/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787828807.176970:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828807.176990:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828807/real 1787828807] req@ffff9e35c498f480 x1874672085688448/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828832 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828807.177002:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f480 x1874672085688448/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828832 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828807.177010:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828807.177013:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828807.177016:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085688448:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787828807.177035:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085688576:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787828807.177050:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828807.177059:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828807/real 1787828807] req@ffff9e35c498f0c0 x1874672085688576/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828832 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828807.177068:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f0c0 x1874672085688576/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828832 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828807.177081:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828807.177084:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828807.177087:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085688576:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787828807.177092:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085688704:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787828807.177104:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828807.177111:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828807/real 1787828807] req@ffff9e35c498e580 x1874672085688704/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828832 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828807.177120:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e580 x1874672085688704/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828832 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828807.177127:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828807.177130:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828807.177132:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085688704:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787828807.177136:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085688832:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787828807.177148:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828807.177155:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828807/real 1787828807] req@ffff9e35c498cf00 x1874672085688832/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828832 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828807.177164:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498cf00 x1874672085688832/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828832 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828807.177171:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828807.177173:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828807.177177:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085688832:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787828807.177182:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085688960:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787828807.177194:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828807.177201:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828807/real 1787828807] req@ffff9e35c498d680 x1874672085688960/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828832 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828807.177210:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498d680 x1874672085688960/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828832 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828807.177218:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828807.177220:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828807.177222:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085688960:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787828807.304007:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828807.304075:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787828807.304268:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828807.304281:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787828808.327915:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828808.327933:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828808.327935:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828808.327975:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787828809.351926:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:3.0:1787828809.351926:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828809.351946:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787828809.352033:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828810.376030:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828810.376043:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828810.376107:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787828810.376156:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828811.399988:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828811.399989:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828811.400116:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828811.400124:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00080000:1.0:1787828812.296011:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828812.296022:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828812.296027:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828812.296038:0:27606:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2247 00000100:00080000:1.0:1787828812.296051:0:27606:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828812.296058:0:27606:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787828812.296110:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4f00 x1874672085689088/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828812.296145:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828812.296148:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828812.296150:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828812.296154:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2247 00000100:00080000:1.0:1787828812.296157:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828812.296161:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828812.296172:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828812.296185:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6d00 x1874672085689216/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787828812.296197:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085689088:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787828812.296197:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828812.296217:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787828812.296220:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828812.296227:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2247 00000100:00080000:1.0:1787828812.296230:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828812.296233:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828812.296237:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828812.296247:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4780 x1874672085689344/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828812.296260:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828812.296262:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828812.296264:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828812.296267:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2247 00000100:00080000:1.0:1787828812.296269:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828812.296271:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828812.296275:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:1.0:1787828812.296285:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5a40 x1874672085689472/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828812.296291:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828812.296294:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828812.296296:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828812.296299:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2247 00000100:00080000:1.0:1787828812.296301:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828812.296303:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828812.296306:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000800:00000100:0.0:1787828812.296310:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828812.296318:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7840 x1874672085689600/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828812.296326:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828812.296329:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828812.296331:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828812.296334:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2247 00000100:00080000:1.0:1787828812.296339:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828812.296342:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828812.296345:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828812.296356:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4000 x1874672085689728/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828812.296366:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828812.296371:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:0.0:1787828812.296373:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828812/real 1787828812] req@ffff9e35da6f4f00 x1874672085689088/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828837 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828812.296374:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828812.296384:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2247 00000100:00080000:1.0:1787828812.296392:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828812.296400:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828812.296406:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828812.296411:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4f00 x1874672085689088/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828837 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828812.296421:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5e00 x1874672085689856/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828812.296433:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828812.296434:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828812.296439:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828812.296443:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828812.296447:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:1.0:1787828812.296450:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2247 00000100:00080000:1.0:1787828812.296455:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787828812.296460:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085689088:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787828812.296462:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828812.296470:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:1.0:1787828812.296479:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7480 x1874672085689984/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828812.296490:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828812.296493:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828812.296494:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787828812.296499:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085689216:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787828812.296500:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2247 00000100:00080000:1.0:1787828812.296505:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828812.296509:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828812.296512:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:1.0:1787828812.296521:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4b40 x1874672085690112/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787828812.296547:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828812.296581:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828812/real 1787828812] req@ffff9e35da6f6d00 x1874672085689216/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828837 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828812.296601:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6d00 x1874672085689216/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828837 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828812.296614:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828812.296618:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828812.296623:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085689216:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787828812.296632:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085689344:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787828812.296659:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828812.296672:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828812/real 1787828812] req@ffff9e35da6f4780 x1874672085689344/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828837 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828812.296686:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4780 x1874672085689344/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828837 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828812.296697:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828812.296700:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828812.296704:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085689344:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787828812.296711:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085689472:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787828812.296731:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828812.296744:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828812/real 1787828812] req@ffff9e35da6f5a40 x1874672085689472/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828837 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828812.296783:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5a40 x1874672085689472/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828837 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828812.296792:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828812.296796:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828812.296799:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085689472:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787828812.296806:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085689600:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787828812.296887:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828812.296918:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828812/real 1787828812] req@ffff9e35da6f7840 x1874672085689600/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828837 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828812.296934:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7840 x1874672085689600/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828837 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828812.296945:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828812.296950:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828812.296954:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085689600:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787828812.296962:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085689728:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787828812.297002:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828812.297010:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828812/real 1787828812] req@ffff9e35da6f4000 x1874672085689728/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828837 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828812.297024:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4000 x1874672085689728/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828837 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828812.297031:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828812.297033:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828812.297035:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085689728:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787828812.297039:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085689856:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787828812.297051:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828812.297058:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828812/real 1787828812] req@ffff9e35da6f5e00 x1874672085689856/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828837 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828812.297066:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5e00 x1874672085689856/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828837 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828812.297072:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828812.297074:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828812.297076:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085689856:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787828812.297080:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085689984:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787828812.297091:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828812.297098:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828812/real 1787828812] req@ffff9e35da6f7480 x1874672085689984/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828837 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828812.297106:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7480 x1874672085689984/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828837 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828812.297117:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828812.297120:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828812.297122:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085689984:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787828812.297143:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085690112:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787828812.297155:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828812.297162:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828812/real 1787828812] req@ffff9e35da6f4b40 x1874672085690112/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828837 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828812.297170:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4b40 x1874672085690112/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828837 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828812.297177:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828812.297179:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828812.297182:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085690112:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828812.423969:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828812.424022:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828812.424054:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:0.0:1787828812.424068:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828813.447943:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828813.447947:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828813.448087:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787828813.448087:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828814.471933:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787828814.471988:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828814.471995:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828814.472007:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828815.495904:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828815.496019:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828815.496032:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787828815.496071:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828816.519934:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787828816.519966:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828816.520016:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787828816.520056:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828817.416006:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828817.416039:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828817.416045:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828817.416061:0:27606:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2252 00000100:00080000:1.0:1787828817.416076:0:27606:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828817.416104:0:27606:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787828817.416163:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f70c0 x1874672085690240/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828817.416226:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828817.416231:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828817.416234:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828817.416239:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2252 00000100:00080000:1.0:1787828817.416244:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828817.416249:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828817.416263:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828817.416293:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6940 x1874672085690368/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828817.416309:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787828817.416310:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085690240:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787828817.416313:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787828817.416317:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828817.416330:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2252 00000100:00080000:1.0:1787828817.416334:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828817.416338:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828817.416342:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828817.416377:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6580 x1874672085690496/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828817.416387:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828817.416391:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828817.416393:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828817.416398:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2252 00000100:00080000:1.0:1787828817.416426:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828817.416431:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828817.416436:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000800:00000100:0.0:1787828817.416440:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828817.416456:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f43c0 x1874672085690624/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828817.416467:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828817.416471:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828817.416473:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828817.416483:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2252 00000100:00080000:1.0:1787828817.416508:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787828817.416531:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828817/real 1787828817] req@ffff9e35da6f70c0 x1874672085690240/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828842 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828817.416534:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828817.416554:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:0.0:1787828817.416584:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f70c0 x1874672085690240/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828842 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828817.416586:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f61c0 x1874672085690752/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828817.416664:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828817.416673:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828817.416676:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828817.416682:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2252 00000100:00080000:0.0:1787828817.416685:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828817.416687:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828817.416700:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:1.0:1787828817.416701:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828817.416720:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828817.416721:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085690240:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1787828817.416750:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4b40 x1874672085690880/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828817.416760:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828817.416764:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787828817.416767:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828817.416777:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2252 00000100:00080000:1.0:1787828817.416782:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787828817.416786:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085690368:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787828817.416790:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828817.416880:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1787828817.416925:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828817.416931:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7480 x1874672085691008/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828817.416949:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828817.416966:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828817.416968:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787828817.416973:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828817/real 1787828817] req@ffff9e35da6f6940 x1874672085690368/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828842 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828817.416978:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2252 00000100:00080000:1.0:1787828817.416997:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828817.417006:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787828817.417014:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6940 x1874672085690368/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828842 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1787828817.417033:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00080000:0.0:1787828817.417041:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828817.417048:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828817.417058:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5e00 x1874672085691136/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787828817.417059:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085690368:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787828817.417106:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085690496:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787828817.417117:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828817.417121:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828817.417124:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828817.417132:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2252 00000100:00080000:1.0:1787828817.417141:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1787828817.417146:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787828817.417153:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828817.417165:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:0.0:1787828817.417169:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828817/real 1787828817] req@ffff9e35da6f6580 x1874672085690496/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828842 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828817.417184:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6580 x1874672085690496/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828842 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828817.417194:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4000 x1874672085691264/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828817.417197:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828817.417231:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828817.417243:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085690496:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787828817.417256:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085690624:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787828817.417295:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828817.417310:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828817/real 1787828817] req@ffff9e35da6f43c0 x1874672085690624/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828842 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828817.417322:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f43c0 x1874672085690624/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828842 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828817.417330:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828817.417333:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828817.417336:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085690624:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787828817.417342:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085690752:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787828817.417360:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828817.417368:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828817/real 1787828817] req@ffff9e35da6f61c0 x1874672085690752/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828842 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828817.417378:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f61c0 x1874672085690752/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828842 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828817.417386:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828817.417388:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828817.417391:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085690752:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787828817.417396:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085690880:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787828817.417410:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828817.417418:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828817/real 1787828817] req@ffff9e35da6f4b40 x1874672085690880/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828842 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828817.417429:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4b40 x1874672085690880/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828842 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828817.417436:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828817.417439:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828817.417441:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085690880:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787828817.417465:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085691008:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787828817.417480:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828817.417508:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828817/real 1787828817] req@ffff9e35da6f7480 x1874672085691008/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828842 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828817.417538:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7480 x1874672085691008/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828842 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828817.417545:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828817.417548:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828817.417551:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085691008:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787828817.417556:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085691136:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787828817.417575:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828817.417584:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828817/real 1787828817] req@ffff9e35da6f5e00 x1874672085691136/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828842 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828817.417594:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5e00 x1874672085691136/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828842 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828817.417608:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828817.417610:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828817.417631:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085691136:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787828817.417636:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085691264:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787828817.417652:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828817.417660:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828817/real 1787828817] req@ffff9e35da6f4000 x1874672085691264/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828842 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828817.417670:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4000 x1874672085691264/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828842 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828817.417678:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828817.417681:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828817.417683:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085691264:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787828817.543886:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787828817.543997:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828817.544009:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828817.544033:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828818.567898:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828818.567898:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787828818.567946:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787828818.567973:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828819.591951:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828819.591951:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828819.592001:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828819.592017:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828820.615910:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828820.615910:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787828820.615936:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787828820.615951:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828821.639927:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828821.639954:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787828821.639961:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828821.640001:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828822.535948:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828822.535957:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828822.535961:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828822.535980:0:27606:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2257 00000100:00080000:0.0:1787828822.535991:0:27606:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828822.535999:0:27606:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787828822.536039:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7192c0 x1874672085691392/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828822.536066:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828822.536070:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828822.536072:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828822.536077:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2257 00000100:00080000:0.0:1787828822.536082:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828822.536088:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828822.536099:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828822.536114:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085691392:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1787828822.536118:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718b40 x1874672085691520/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828822.536132:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828822.536135:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787828822.536136:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828822.536140:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2257 00000100:00080000:0.0:1787828822.536144:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828822.536147:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828822.536150:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828822.536160:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718f00 x1874672085691648/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828822.536167:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828822.536169:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787828822.536171:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828822.536175:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2257 00000100:00080000:0.0:1787828822.536178:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828822.536182:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:2.0:1787828822.536188:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:0.0:1787828822.536197:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:0.0:1787828822.536210:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a940 x1874672085691776/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828822.536218:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828822.536220:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787828822.536222:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828822.536225:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2257 00000100:00080000:0.0:1787828822.536227:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828822.536231:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828822.536235:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:2.0:1787828822.536239:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828822/real 1787828822] req@ffff9e35da7192c0 x1874672085691392/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828847 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828822.536249:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a1c0 x1874672085691904/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828822.536258:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787828822.536262:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7192c0 x1874672085691392/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828847 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828822.536263:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828822.536267:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828822.536275:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2257 00000100:00080000:2.0:1787828822.536281:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828822.536288:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828822.536289:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:0.0:1787828822.536296:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787828822.536298:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085691392:192.168.203.141@tcp:250:kworker.0 00000008:00100000:0.0:1787828822.536301:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828822.536315:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7183c0 x1874672085692032/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828822.536326:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085691520:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787828822.536333:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828822.536339:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787828822.536342:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828822.536348:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2257 00000800:00000100:2.0:1787828822.536354:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787828822.536357:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828822.536366:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787828822.536368:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828822/real 1787828822] req@ffff9e35da718b40 x1874672085691520/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828847 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1787828822.536373:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828822.536385:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718b40 x1874672085691520/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828847 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828822.536393:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a580 x1874672085692160/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828822.536396:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828822.536406:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:0.0:1787828822.536413:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787828822.536414:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085691520:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787828822.536423:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787828822.536426:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787828822.536428:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085691648:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787828822.536433:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2257 00000100:00080000:0.0:1787828822.536438:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828822.536441:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828822.536446:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000800:00000100:2.0:1787828822.536452:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828822.536481:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da719a40 x1874672085692288/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828822.536484:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828822/real 1787828822] req@ffff9e35da718f00 x1874672085691648/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828847 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828822.536537:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787828822.536542:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672085691648/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828847 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828822.536544:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787828822.536549:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828822.536563:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2257 00000100:00080000:2.0:1787828822.536564:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828822.536574:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:0.0:1787828822.536575:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787828822.536579:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085691648:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787828822.536588:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787828822.536590:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085691776:192.168.203.141@tcp:8:kworker.0 00000008:00100000:0.0:1787828822.536605:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000800:00000100:2.0:1787828822.536623:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828822.536631:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718780 x1874672085692416/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828822.536638:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828822/real 1787828822] req@ffff9e35da71a940 x1874672085691776/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828847 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828822.536662:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672085691776/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828847 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828822.536671:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828822.536674:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828822.536677:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085691776:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828822.536682:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085691904:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828822.536696:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828822.536704:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828822/real 1787828822] req@ffff9e35da71a1c0 x1874672085691904/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828847 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828822.536713:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a1c0 x1874672085691904/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828847 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828822.536720:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828822.536722:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828822.536725:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085691904:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828822.536745:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085692032:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828822.536758:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828822.536766:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828822/real 1787828822] req@ffff9e35da7183c0 x1874672085692032/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828847 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828822.536776:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672085692032/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828847 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828822.536805:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828822.536808:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828822.536812:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085692032:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828822.536818:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085692160:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828822.536889:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828822.536898:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828822/real 1787828822] req@ffff9e35da71a580 x1874672085692160/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828847 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828822.536907:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672085692160/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828847 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828822.536913:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828822.536915:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828822.536918:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085692160:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828822.536922:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085692288:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828822.536935:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828822.536941:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828822/real 1787828822] req@ffff9e35da719a40 x1874672085692288/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828847 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828822.536949:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672085692288/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828847 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828822.536955:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828822.536957:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828822.536959:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085692288:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828822.536963:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085692416:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828822.536974:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828822.536989:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828822/real 1787828822] req@ffff9e35da718780 x1874672085692416/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828847 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828822.536997:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718780 x1874672085692416/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828847 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828822.537003:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828822.537005:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828822.537007:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085692416:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828822.663906:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787828822.663909:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828822.663923:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828822.663939:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828823.687926:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787828823.687938:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828823.687942:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828823.687954:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787828824.711917:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828824.711917:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828824.711918:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828824.711997:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828825.735873:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787828825.735896:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828825.735929:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828825.735962:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828826.759884:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787828826.759887:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828826.759918:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828826.759930:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828827.655970:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828827.655980:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828827.655985:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828827.655996:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2263 00000100:00080000:2.0:1787828827.656007:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828827.656015:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787828827.656052:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c780 x1874672085692544/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828827.656077:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828827.656080:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828827.656082:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828827.656084:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2263 00000100:00080000:2.0:1787828827.656088:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828827.656091:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828827.656100:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828827.656113:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085692544:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1787828827.656113:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e940 x1874672085692672/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828827.656128:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828827.656130:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787828827.656131:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828827.656134:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2263 00000100:00080000:2.0:1787828827.656136:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828827.656139:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828827.656142:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828827.656152:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498fc00 x1874672085692800/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828827.656158:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828827.656160:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787828827.656161:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828827.656164:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2263 00000100:00080000:2.0:1787828827.656167:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828827.656169:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828827.656174:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000800:00000100:0.0:1787828827.656175:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828827.656187:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498de00 x1874672085692928/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828827.656194:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828827.656196:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787828827.656197:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828827.656200:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2263 00000100:00080000:2.0:1787828827.656210:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787828827.656212:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828827/real 1787828827] req@ffff9e35c498c780 x1874672085692544/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828852 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828827.656216:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828827.656223:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:0.0:1787828827.656234:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c780 x1874672085692544/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828852 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828827.656237:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498ed00 x1874672085693056/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828827.656256:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828827.656257:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828827.656263:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828827.656265:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:2.0:1787828827.656266:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787828827.656272:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085692544:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1787828827.656273:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2263 00000100:00080000:2.0:1787828827.656281:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828827.656284:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828827.656287:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828827.656294:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c3c0 x1874672085693184/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787828827.656301:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085692672:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1787828827.656302:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828827.656308:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787828827.656309:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828827.656313:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2263 00000100:00080000:2.0:1787828827.656317:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828827.656320:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828827.656324:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1787828827.656325:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828827.656337:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c780 x1874672085693312/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787828827.656339:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828827/real 1787828827] req@ffff9e35c498e940 x1874672085692672/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828852 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828827.656361:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828827.656368:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787828827.656370:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828827.656379:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2263 00000100:00080000:2.0:1787828827.656381:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787828827.656383:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e940 x1874672085692672/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828852 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828827.656387:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828827.656393:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00080000:0.0:1787828827.656398:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828827.656403:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828827.656406:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498d680 x1874672085693440/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787828827.656410:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085692672:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787828827.656437:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085692800:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1787828827.656442:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828827.656446:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787828827.656448:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828827.656453:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2263 00000100:00080000:2.0:1787828827.656456:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1787828827.656457:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1787828827.656463:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828827.656469:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:0.0:1787828827.656470:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828827/real 1787828827] req@ffff9e35c498fc00 x1874672085692800/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828852 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828827.656489:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498fc00 x1874672085692800/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828852 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828827.656489:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cf00 x1874672085693568/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828827.656522:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828827.656527:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828827.656530:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085692800:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787828827.656534:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085692928:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787828827.656547:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828827.656553:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828827/real 1787828827] req@ffff9e35c498de00 x1874672085692928/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828852 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828827.656561:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498de00 x1874672085692928/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828852 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828827.656571:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828827.656573:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828827.656575:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085692928:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787828827.656578:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085693056:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787828827.656587:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828827.656592:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828827/real 1787828827] req@ffff9e35c498ed00 x1874672085693056/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828852 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828827.656599:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498ed00 x1874672085693056/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828852 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828827.656605:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828827.656607:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828827.656608:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085693056:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787828827.656623:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085693184:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787828827.656633:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828827.656639:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828827/real 1787828827] req@ffff9e35c498c3c0 x1874672085693184/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828852 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828827.656646:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c3c0 x1874672085693184/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828852 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828827.656652:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828827.656653:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828827.656656:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085693184:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787828827.656659:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085693312:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787828827.656669:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828827.656674:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828827/real 1787828827] req@ffff9e35c498c780 x1874672085693312/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828852 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828827.656681:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c780 x1874672085693312/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828852 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828827.656686:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828827.656688:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828827.656690:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085693312:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787828827.656693:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085693440:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787828827.656702:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828827.656707:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828827/real 1787828827] req@ffff9e35c498d680 x1874672085693440/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828852 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828827.656713:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498d680 x1874672085693440/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828852 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828827.656718:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828827.656720:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828827.656723:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085693440:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787828827.656726:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085693568:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787828827.656735:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828827.656741:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828827/real 1787828827] req@ffff9e35c498cf00 x1874672085693568/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828852 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828827.656748:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498cf00 x1874672085693568/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828852 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828827.656753:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828827.656755:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828827.656757:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085693568:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787828827.783846:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828827.783882:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828827.783933:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828827.783937:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828828.807865:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787828828.807872:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787828828.807890:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828828.807938:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828829.831938:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:3.0:1787828829.831946:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787828829.831978:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828829.832055:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828830.855953:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787828830.855960:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828830.855987:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828830.856007:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787828831.879998:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787828831.879999:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828831.880002:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:0.0:1787828831.880097:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828832.776031:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828832.776045:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828832.776050:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828832.776067:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2268 00000100:00080000:1.0:1787828832.776084:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828832.776108:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787828832.776173:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7840 x1874672085693696/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828832.776229:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828832.776233:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828832.776235:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828832.776238:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2268 00000100:00080000:1.0:1787828832.776242:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828832.776246:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828832.776257:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828832.776288:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5a40 x1874672085693824/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828832.776301:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828832.776307:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:0.0:1787828832.776309:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085693696:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787828832.776310:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828832.776322:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2268 00000100:00080000:1.0:1787828832.776327:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828832.776330:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828832.776335:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828832.776361:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4780 x1874672085693952/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828832.776372:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828832.776375:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828832.776377:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828832.776381:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2268 00000100:00080000:1.0:1787828832.776411:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828832.776415:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828832.776447:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000800:00000100:0.0:1787828832.776452:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828832.776460:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6d00 x1874672085694080/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828832.776469:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828832.776473:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828832.776474:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828832.776478:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2268 00000100:00080000:1.0:1787828832.776486:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828832.776490:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828832.776493:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:1.0:1787828832.776503:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4f00 x1874672085694208/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787828832.776508:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828832/real 1787828832] req@ffff9e35da6f7840 x1874672085693696/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828857 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828832.776515:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828832.776523:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828832.776527:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828832.776533:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2268 00000100:00080000:1.0:1787828832.776536:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828832.776539:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787828832.776543:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7840 x1874672085693696/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828857 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1787828832.776545:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1787828832.776556:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828832.776561:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828832.776568:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085693696:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1787828832.776568:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4000 x1874672085694336/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828832.776584:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828832.776587:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787828832.776589:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828832.776592:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2268 00000100:00080000:1.0:1787828832.776595:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828832.776597:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828832.776601:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828832.776613:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085693824:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828832.776613:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5e00 x1874672085694464/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828832.776634:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828832.776637:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828832.776638:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828832.776642:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2268 00000100:00080000:1.0:1787828832.776645:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828832.776647:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828832.776652:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:1.0:1787828832.776660:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7480 x1874672085694592/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787828832.776667:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787828832.776675:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828832.776679:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828832.776680:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828832.776694:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2268 00000100:00080000:1.0:1787828832.776698:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787828832.776699:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828832/real 1787828832] req@ffff9e35da6f5a40 x1874672085693824/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828857 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828832.776703:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828832.776711:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:0.0:1787828832.776721:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5a40 x1874672085693824/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828857 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828832.776724:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4b40 x1874672085694720/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828832.776751:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828832.776759:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828832.776764:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085693824:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787828832.776775:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085693952:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787828832.776802:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828832.776840:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828832/real 1787828832] req@ffff9e35da6f4780 x1874672085693952/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828857 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828832.776850:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4780 x1874672085693952/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828857 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828832.776857:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828832.776860:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828832.776863:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085693952:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787828832.776868:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085694080:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787828832.776885:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828832.776893:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828832/real 1787828832] req@ffff9e35da6f6d00 x1874672085694080/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828857 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828832.776907:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6d00 x1874672085694080/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828857 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828832.776913:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828832.776915:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828832.776917:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085694080:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787828832.776921:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085694208:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787828832.776936:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828832.776948:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828832/real 1787828832] req@ffff9e35da6f4f00 x1874672085694208/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828857 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828832.776962:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4f00 x1874672085694208/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828857 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828832.776973:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828832.776977:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828832.776981:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085694208:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787828832.776988:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085694336:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787828832.777008:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828832.777018:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828832/real 1787828832] req@ffff9e35da6f4000 x1874672085694336/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828857 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828832.777027:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4000 x1874672085694336/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828857 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828832.777038:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828832.777041:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828832.777043:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085694336:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787828832.777064:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085694464:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787828832.777077:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828832.777084:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828832/real 1787828832] req@ffff9e35da6f5e00 x1874672085694464/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828857 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828832.777093:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5e00 x1874672085694464/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828857 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828832.777101:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828832.777104:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828832.777108:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085694464:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787828832.777115:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085694592:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787828832.777146:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828832.777160:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828832/real 1787828832] req@ffff9e35da6f7480 x1874672085694592/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828857 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828832.777174:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7480 x1874672085694592/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828857 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828832.777181:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828832.777183:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828832.777187:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085694592:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787828832.777191:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085694720:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787828832.777207:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828832.777215:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828832/real 1787828832] req@ffff9e35da6f4b40 x1874672085694720/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828857 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828832.777223:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4b40 x1874672085694720/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828857 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828832.777229:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828832.777231:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828832.777233:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085694720:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787828832.903944:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828832.903995:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:3.0:1787828832.904007:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828832.904102:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828833.927881:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828833.927887:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:3.0:1787828833.927891:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828833.927975:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828834.951904:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828834.951904:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:3.0:1787828834.951913:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787828834.952020:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828835.975927:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828835.975927:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:3.0:1787828835.975927:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828835.976004:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828836.999903:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828836.999904:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828836.999931:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828836.999968:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828837.895878:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828837.895886:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828837.895890:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828837.895898:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2273 00000100:00080000:1.0:1787828837.895907:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828837.895913:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787828837.895944:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f61c0 x1874672085694848/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828837.895967:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828837.895970:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828837.895971:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828837.895975:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2273 00000100:00080000:1.0:1787828837.895979:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828837.895983:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828837.895993:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828837.896007:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f43c0 x1874672085694976/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787828837.896016:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085694848:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787828837.896025:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828837.896029:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787828837.896034:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828837.896038:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2273 00000100:00080000:1.0:1787828837.896042:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828837.896046:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828837.896050:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828837.896060:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6580 x1874672085695104/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828837.896067:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828837.896069:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828837.896071:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828837.896073:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2273 00000100:00080000:1.0:1787828837.896084:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828837.896087:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828837.896090:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000800:00000100:0.0:1787828837.896095:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828837.896099:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6940 x1874672085695232/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828837.896107:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828837.896110:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828837.896112:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828837.896115:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2273 00000100:00080000:1.0:1787828837.896118:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828837.896121:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828837.896124:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:1.0:1787828837.896144:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f70c0 x1874672085695360/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787828837.896155:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828837/real 1787828837] req@ffff9e35da6f61c0 x1874672085694848/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828862 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828837.896158:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828837.896165:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828837.896167:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828837.896176:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2273 00000100:00080000:1.0:1787828837.896179:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828837.896182:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828837.896186:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828837.896193:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f61c0 x1874672085694848/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828862 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828837.896195:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4b40 x1874672085695488/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828837.896225:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828837.896228:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787828837.896229:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828837.896230:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828837.896241:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2273 00000100:00080000:0.0:1787828837.896242:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:1.0:1787828837.896245:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787828837.896262:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085694848:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787828837.896263:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828837.896269:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828837.896277:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7480 x1874672085695616/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828837.896283:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828837.896285:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828837.896286:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828837.896289:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2273 00000100:00080000:1.0:1787828837.896292:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828837.896296:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787828837.896298:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085694976:192.168.203.141@tcp:38:kworker.0 00000008:00100000:1.0:1787828837.896300:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:1.0:1787828837.896311:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5e00 x1874672085695744/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828837.896318:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828837.896321:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828837.896323:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1787828837.896325:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787828837.896329:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2273 00000100:00080000:1.0:1787828837.896334:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828837.896336:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828837.896340:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:0.0:1787828837.896343:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828837/real 1787828837] req@ffff9e35da6f43c0 x1874672085694976/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828862 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828837.896366:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4000 x1874672085695872/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787828837.896373:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f43c0 x1874672085694976/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828862 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828837.896392:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828837.896395:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828837.896397:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085694976:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787828837.896405:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085695104:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787828837.896421:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828837.896429:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828837/real 1787828837] req@ffff9e35da6f6580 x1874672085695104/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828862 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828837.896436:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6580 x1874672085695104/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828862 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828837.896450:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828837.896452:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828837.896454:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085695104:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787828837.896458:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085695232:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787828837.896468:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828837.896474:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828837/real 1787828837] req@ffff9e35da6f6940 x1874672085695232/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828862 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828837.896482:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6940 x1874672085695232/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828862 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828837.896488:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828837.896489:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828837.896491:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085695232:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787828837.896494:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085695360:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787828837.896504:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828837.896510:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828837/real 1787828837] req@ffff9e35da6f70c0 x1874672085695360/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828862 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828837.896518:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f70c0 x1874672085695360/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828862 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828837.896524:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828837.896526:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828837.896527:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085695360:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787828837.896531:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085695488:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787828837.896540:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828837.896546:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828837/real 1787828837] req@ffff9e35da6f4b40 x1874672085695488/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828862 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828837.896553:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4b40 x1874672085695488/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828862 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828837.896560:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828837.896562:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828837.896564:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085695488:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787828837.896568:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085695616:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787828837.896577:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828837.896583:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828837/real 1787828837] req@ffff9e35da6f7480 x1874672085695616/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828862 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828837.896589:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7480 x1874672085695616/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828862 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828837.896595:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828837.896597:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828837.896598:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085695616:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787828837.896613:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085695744:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787828837.896624:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828837.896630:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828837/real 1787828837] req@ffff9e35da6f5e00 x1874672085695744/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828862 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828837.896638:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5e00 x1874672085695744/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828862 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828837.896654:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828837.896656:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828837.896659:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085695744:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787828837.896662:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085695872:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787828837.896671:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828837.896677:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828837/real 1787828837] req@ffff9e35da6f4000 x1874672085695872/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828862 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828837.896684:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4000 x1874672085695872/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828862 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828837.896689:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828837.896691:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828837.896693:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085695872:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828838.023948:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:3.0:1787828838.023963:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787828838.023972:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787828838.024062:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828839.047908:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828839.047927:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828839.047960:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828839.047960:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828840.071902:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787828840.071962:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828840.071969:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828840.072011:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787828841.095867:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828841.095905:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828841.095910:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:0.0:1787828841.095936:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828842.119894:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787828842.119903:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828842.119926:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828842.120076:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00080000:2.0:1787828843.015884:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828843.015891:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828843.015895:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828843.015903:0:27606:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2278 00000100:00080000:2.0:1787828843.015912:0:27606:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828843.015918:0:27606:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787828843.015948:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e580 x1874672085696000/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828843.015971:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828843.015974:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828843.015975:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828843.015978:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2278 00000100:00080000:2.0:1787828843.015980:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828843.015983:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828843.015991:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828843.016001:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f0c0 x1874672085696128/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828843.016009:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828843.016013:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787828843.016014:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828843.016017:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2278 00000100:00080000:2.0:1787828843.016020:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828843.016022:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828843.016025:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828843.016033:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f480 x1874672085696256/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828843.016042:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828843.016046:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:0.0:1787828843.016047:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085696000:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1787828843.016049:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828843.016059:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2278 00000100:00080000:2.0:1787828843.016068:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828843.016071:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828843.016074:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:2.0:1787828843.016086:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e1c0 x1874672085696384/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828843.016095:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828843.016097:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787828843.016099:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828843.016102:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2278 00000100:00080000:2.0:1787828843.016104:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828843.016106:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828843.016111:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:2.0:1787828843.016121:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cb40 x1874672085696512/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828843.016127:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828843.016130:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828843.016131:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828843.016133:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2278 00000100:00080000:2.0:1787828843.016136:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828843.016139:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828843.016141:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828843.016147:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c000 x1874672085696640/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787828843.016158:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1787828843.016161:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828843.016166:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787828843.016169:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828843.016176:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2278 00000100:00080000:2.0:1787828843.016180:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828843.016182:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828843.016187:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828843.016200:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f840 x1874672085696768/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828843.016207:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828843.016210:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787828843.016212:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828843.016214:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2278 00000100:00080000:2.0:1787828843.016218:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787828843.016223:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828843/real 1787828843] req@ffff9e35c498e580 x1874672085696000/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828868 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828843.016228:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828843.016239:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:2.0:1787828843.016259:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cf00 x1874672085696896/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787828843.016263:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e580 x1874672085696000/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828868 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828843.016308:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828843.016326:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787828843.016327:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828843.016330:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828843.016336:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:2.0:1787828843.016337:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2278 00000100:00080000:2.0:1787828843.016352:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787828843.016355:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085696000:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1787828843.016363:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828843.016371:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:0.0:1787828843.016375:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085696128:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828843.016385:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498d680 x1874672085697024/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787828843.016405:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828843.016422:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828843/real 1787828843] req@ffff9e35c498f0c0 x1874672085696128/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828868 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828843.016431:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f0c0 x1874672085696128/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828868 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828843.016438:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828843.016441:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828843.016443:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085696128:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787828843.016469:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085696256:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787828843.016481:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828843.016488:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828843/real 1787828843] req@ffff9e35c498f480 x1874672085696256/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828868 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828843.016496:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f480 x1874672085696256/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828868 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828843.016502:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828843.016504:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828843.016507:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085696256:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787828843.016511:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085696384:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787828843.016523:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828843.016529:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828843/real 1787828843] req@ffff9e35c498e1c0 x1874672085696384/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828868 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828843.016537:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e1c0 x1874672085696384/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828868 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828843.016542:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828843.016544:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828843.016546:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085696384:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787828843.016550:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085696512:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787828843.016559:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828843.016565:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828843/real 1787828843] req@ffff9e35c498cb40 x1874672085696512/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828868 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828843.016572:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498cb40 x1874672085696512/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828868 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828843.016578:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828843.016580:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828843.016582:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085696512:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787828843.016586:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085696640:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787828843.016595:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828843.016601:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828843/real 1787828843] req@ffff9e35c498c000 x1874672085696640/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828868 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828843.016614:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c000 x1874672085696640/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828868 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828843.016619:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828843.016621:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828843.016623:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085696640:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787828843.016627:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085696768:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787828843.016636:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828843.016642:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828843/real 1787828843] req@ffff9e35c498f840 x1874672085696768/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828868 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828843.016648:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f840 x1874672085696768/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828868 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828843.016653:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828843.016655:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828843.016657:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085696768:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787828843.016668:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085696896:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787828843.016679:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828843.016685:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828843/real 1787828843] req@ffff9e35c498cf00 x1874672085696896/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828868 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828843.016692:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498cf00 x1874672085696896/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828868 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828843.016703:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828843.016705:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828843.016707:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085696896:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787828843.016711:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085697024:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787828843.016721:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828843.016726:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828843/real 1787828843] req@ffff9e35c498d680 x1874672085697024/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828868 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828843.016734:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498d680 x1874672085697024/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828868 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828843.016739:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828843.016741:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828843.016743:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085697024:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828843.143880:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:3.0:1787828843.143881:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828843.143926:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828843.143943:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828844.167896:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:3.0:1787828844.167905:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828844.167910:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787828844.167997:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828845.191867:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828845.191950:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828845.191969:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828845.191977:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828846.215893:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787828846.215898:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787828846.215940:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828846.215942:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787828847.239952:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828847.239953:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828847.239954:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787828847.240076:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828848.135910:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828848.135918:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828848.135922:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828848.135931:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2283 00000100:00080000:1.0:1787828848.135941:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828848.135947:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787828848.135979:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4f00 x1874672085697152/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828848.136006:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828848.136010:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828848.136011:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828848.136014:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2283 00000100:00080000:1.0:1787828848.136017:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828848.136019:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828848.136027:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828848.136036:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6d00 x1874672085697280/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828848.136044:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828848.136047:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787828848.136048:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828848.136051:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2283 00000100:00100000:0.0:1787828848.136058:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085697152:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787828848.136060:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828848.136069:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828848.136074:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828848.136083:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4780 x1874672085697408/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828848.136091:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828848.136093:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828848.136095:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828848.136098:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2283 00000100:00080000:1.0:1787828848.136119:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828848.136122:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828848.136125:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000800:00000100:0.0:1787828848.136130:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828848.136139:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5a40 x1874672085697536/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828848.136148:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828848.136150:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828848.136151:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828848.136154:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2283 00000100:00080000:1.0:1787828848.136157:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828848.136159:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828848.136163:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:0.0:1787828848.136170:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828848/real 1787828848] req@ffff9e35da6f4f00 x1874672085697152/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828873 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828848.136174:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7840 x1874672085697664/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828848.136196:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828848.136199:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828848.136202:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787828848.136203:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4f00 x1874672085697152/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828873 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828848.136208:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2283 00000100:00080000:1.0:1787828848.136213:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828848.136219:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828848.136223:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828848.136233:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4000 x1874672085697792/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828848.136241:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828848.136251:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:1.0:1787828848.136254:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787828848.136260:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085697152:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787828848.136260:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787828848.136262:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828848.136272:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2283 00000100:00080000:1.0:1787828848.136274:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787828848.136290:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085697280:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787828848.136291:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828848.136299:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828848.136312:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5e00 x1874672085697920/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828848.136319:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1787828848.136324:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787828848.136325:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828848.136333:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787828848.136339:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828848/real 1787828848] req@ffff9e35da6f6d00 x1874672085697280/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828873 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828848.136339:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2283 00000100:00080000:1.0:1787828848.136344:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828848.136351:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828848.136356:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:0.0:1787828848.136358:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6d00 x1874672085697280/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828873 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828848.136367:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1787828848.136368:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7480 x1874672085698048/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828848.136371:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828848.136378:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085697280:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787828848.136388:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828848.136391:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828848.136393:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828848.136396:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2283 00000100:00080000:1.0:1787828848.136399:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828848.136402:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828848.136404:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:0.0:1787828848.136412:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085697408:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828848.136414:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4b40 x1874672085698176/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787828848.136433:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828848.136443:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828848/real 1787828848] req@ffff9e35da6f4780 x1874672085697408/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828873 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828848.136452:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4780 x1874672085697408/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828873 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828848.136458:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828848.136460:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828848.136463:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085697408:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787828848.136467:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085697536:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787828848.136486:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828848.136494:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828848/real 1787828848] req@ffff9e35da6f5a40 x1874672085697536/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828873 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828848.136502:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5a40 x1874672085697536/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828873 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828848.136508:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828848.136510:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828848.136512:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085697536:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787828848.136516:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085697664:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787828848.136527:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828848.136532:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828848/real 1787828848] req@ffff9e35da6f7840 x1874672085697664/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828873 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828848.136540:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7840 x1874672085697664/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828873 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828848.136546:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828848.136547:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828848.136549:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085697664:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787828848.136553:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085697792:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787828848.136563:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828848.136569:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828848/real 1787828848] req@ffff9e35da6f4000 x1874672085697792/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828873 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828848.136577:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4000 x1874672085697792/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828873 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828848.136582:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828848.136584:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828848.136586:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085697792:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787828848.136589:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085697920:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787828848.136599:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828848.136605:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828848/real 1787828848] req@ffff9e35da6f5e00 x1874672085697920/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828873 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828848.136614:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5e00 x1874672085697920/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828873 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828848.136619:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828848.136621:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828848.136622:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085697920:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787828848.136626:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085698048:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787828848.136636:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828848.136642:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828848/real 1787828848] req@ffff9e35da6f7480 x1874672085698048/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828873 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828848.136649:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7480 x1874672085698048/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828873 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828848.136660:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828848.136662:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828848.136663:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085698048:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787828848.136684:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085698176:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787828848.136695:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828848.136702:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828848/real 1787828848] req@ffff9e35da6f4b40 x1874672085698176/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828873 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828848.136709:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4b40 x1874672085698176/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828873 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828848.136715:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828848.136717:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828848.136719:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085698176:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828848.263874:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828848.263909:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828848.263919:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828848.263945:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828849.287936:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787828849.287955:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828849.287962:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828849.287980:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828850.311950:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828850.311962:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787828850.311968:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787828850.312036:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828851.335914:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828851.335924:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828851.335935:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828851.335940:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828852.359946:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828852.359992:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828852.359996:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828852.360000:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787828853.255888:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828853.255897:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828853.255900:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828853.255910:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2288 00000100:00080000:0.0:1787828853.255924:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828853.255931:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787828853.255963:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718000 x1874672085698304/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828853.255993:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828853.255998:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828853.256000:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828853.256006:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2288 00000100:00080000:0.0:1787828853.256010:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828853.256016:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828853.256028:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828853.256037:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085698304:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1787828853.256047:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718780 x1874672085698432/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828853.256058:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828853.256061:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787828853.256062:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828853.256066:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2288 00000100:00080000:0.0:1787828853.256069:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828853.256072:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828853.256077:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828853.256086:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da719a40 x1874672085698560/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828853.256093:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828853.256096:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787828853.256097:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828853.256100:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2288 00000100:00080000:0.0:1787828853.256104:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828853.256106:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828853.256110:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:0.0:1787828853.256120:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a580 x1874672085698688/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828853.256127:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828853.256130:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787828853.256132:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828853.256136:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2288 00000100:00080000:0.0:1787828853.256137:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828853.256139:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828853.256142:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:1.0:1787828853.256147:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085698432:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787828853.256153:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7183c0 x1874672085698816/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828853.256161:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828853.256163:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828853.256165:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828853.256168:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2288 00000100:00080000:0.0:1787828853.256172:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828853.256175:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787828853.256179:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085698560:192.168.203.141@tcp:38:kworker.0 00000008:00100000:0.0:1787828853.256181:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828853.256194:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a1c0 x1874672085698944/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828853.256205:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085698688:192.168.203.141@tcp:8:kworker.0 00000100:00080000:0.0:1787828853.256209:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828853.256212:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787828853.256214:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828853.256219:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2288 00000100:00080000:0.0:1787828853.256223:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828853.256226:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787828853.256229:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085698816:192.168.203.141@tcp:8:kworker.0 00000008:00100000:0.0:1787828853.256230:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828853.256261:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a940 x1874672085699072/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828853.256271:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787828853.256274:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085698944:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787828853.256275:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787828853.256278:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828853.256286:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2288 00000100:00080000:0.0:1787828853.256289:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828853.256293:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828853.256299:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:1.0:1787828853.256306:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085699072:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787828853.256312:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718f00 x1874672085699200/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828853.256322:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828853.256325:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787828853.256327:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828853.256330:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2288 00000100:00100000:1.0:1787828853.256334:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085699200:192.168.203.141@tcp:8:kworker.0 00000100:00080000:0.0:1787828853.256336:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828853.256343:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828853.256347:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:0.0:1787828853.256359:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718b40 x1874672085699328/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828853.256417:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085699328:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828853.383890:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787828853.383905:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828853.383918:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828853.383925:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828854.407882:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828854.407896:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787828854.407908:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828854.407911:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828855.431934:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828855.431958:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828855.431980:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828855.431994:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1787828856.328211:0:2398:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.141:988 00000400:00000100:3.0:1787828856.328295:0:2398:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.141@tcp at host 192.168.203.141 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1787828856.328318:0:2398:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1787828856.328381:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000800:00000100:3.0:1787828856.328399:0:2398:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1787828856.328411:0:2398:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1787828856.328423:0:2398:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1787828856.328432:0:2398:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1787828856.328453:0:2398:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1787828856.328462:0:2398:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1787828856.328472:0:2398:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:1.0:1787828856.328582:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828853/real 1787828856] req@ffff9e35da718b40 x1874672085699328/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828878 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828856.328614:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718b40 x1874672085699328/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828878 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828856.328630:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828856.328637:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828856.328643:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085699328:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828856.328652:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828853/real 1787828856] req@ffff9e35da718f00 x1874672085699200/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828878 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828856.328670:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672085699200/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828878 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828856.328682:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828856.328686:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828856.328690:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085699200:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828856.328698:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828853/real 1787828856] req@ffff9e35da71a940 x1874672085699072/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828878 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828856.328713:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672085699072/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828878 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828856.328722:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828856.328726:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828856.328730:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085699072:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828856.328749:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828853/real 1787828856] req@ffff9e35da7183c0 x1874672085698816/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828878 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828856.328764:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672085698816/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828878 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828856.328774:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828856.328779:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828856.328783:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085698816:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828856.328791:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828853/real 1787828856] req@ffff9e35da71a1c0 x1874672085698944/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828878 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828856.328861:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a1c0 x1874672085698944/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828878 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828856.328870:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828856.328875:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828856.328879:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085698944:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828856.328886:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828853/real 1787828856] req@ffff9e35da718780 x1874672085698432/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828878 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828856.328900:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718780 x1874672085698432/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828878 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828856.328909:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828856.328912:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828856.328916:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085698432:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828856.328922:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828853/real 1787828856] req@ffff9e35da719a40 x1874672085698560/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828878 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828856.328935:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672085698560/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828878 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828856.328942:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828856.328946:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828856.328950:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085698560:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828856.328956:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828853/real 1787828856] req@ffff9e35da71a580 x1874672085698688/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828878 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828856.328967:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672085698688/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828878 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828856.328974:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828856.328978:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828856.328981:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085698688:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828856.328988:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828853/real 1787828856] req@ffff9e35da718000 x1874672085698304/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828878 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828856.329001:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718000 x1874672085698304/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828878 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828856.329009:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828856.329012:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828856.329015:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085698304:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1787828856.455941:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828856.455943:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828856.455952:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828856.456013:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828857.479942:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828857.479944:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828857.479946:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828857.480019:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828858.376043:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828858.376053:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828858.376058:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828858.376068:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2293 00000100:00080000:0.0:1787828858.376079:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828858.376086:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787828858.376131:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7192c0 x1874672085699456/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828858.376158:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828858.376161:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828858.376163:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828858.376167:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2293 00000100:00080000:0.0:1787828858.376170:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828858.376174:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828858.376183:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828858.376241:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718000 x1874672085699584/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828858.376247:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085699456:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1787828858.376255:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828858.376259:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787828858.376261:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828858.376265:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2293 00000100:00080000:0.0:1787828858.376268:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828858.376271:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828858.376275:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828858.376286:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a580 x1874672085699712/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828858.376294:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828858.376297:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787828858.376298:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828858.376301:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2293 00000100:00080000:0.0:1787828858.376306:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828858.376309:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828858.376313:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000800:00000100:1.0:1787828858.376317:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828858.376338:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da719a40 x1874672085699840/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828858.376350:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828858.376354:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787828858.376357:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828858.376362:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2293 00000100:00080000:0.0:1787828858.376368:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1787828858.376371:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828858/real 1787828858] req@ffff9e35da7192c0 x1874672085699456/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828883 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828858.376376:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828858.376385:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:1.0:1787828858.376401:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7192c0 x1874672085699456/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828883 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828858.376411:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718780 x1874672085699968/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828858.376420:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828858.376429:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828858.376430:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:0.0:1787828858.376438:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:1.0:1787828858.376440:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085699456:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1787828858.376442:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828858.376454:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2293 00000100:00080000:0.0:1787828858.376459:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828858.376464:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828858.376470:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828858.376477:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085699584:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787828858.376487:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a1c0 x1874672085700096/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828858.376506:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1787828858.376511:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787828858.376514:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787828858.376519:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828858.376525:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2293 00000100:00080000:0.0:1787828858.376530:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1787828858.376532:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828858/real 1787828858] req@ffff9e35da718000 x1874672085699584/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828883 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828858.376538:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828858.376551:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828858.376559:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718000 x1874672085699584/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828883 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828858.376571:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7183c0 x1874672085700224/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828858.376575:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828858.376587:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828858.376595:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085699584:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828858.376611:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085699712:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787828858.376614:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828858.376621:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787828858.376624:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828858.376632:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2293 00000100:00080000:0.0:1787828858.376638:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1787828858.376643:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787828858.376649:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828858.376658:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:1.0:1787828858.376661:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828858/real 1787828858] req@ffff9e35da71a580 x1874672085699712/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828883 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828858.376678:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a940 x1874672085700352/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828858.376681:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672085699712/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828883 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828858.376710:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828858.376712:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828858.376720:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828858.376721:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:0.0:1787828858.376727:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787828858.376730:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085699712:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787828858.376736:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2293 00000100:00080000:0.0:1787828858.376743:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828858.376752:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787828858.376752:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085699840:192.168.203.141@tcp:8:kworker.0 00000008:00100000:0.0:1787828858.376763:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:0.0:1787828858.376783:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718f00 x1874672085700480/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787828858.376792:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828858.376841:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828858/real 1787828858] req@ffff9e35da719a40 x1874672085699840/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828883 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828858.376856:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672085699840/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828883 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828858.376866:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828858.376870:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828858.376874:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085699840:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828858.376881:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085699968:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828858.376905:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828858.376918:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828858/real 1787828858] req@ffff9e35da718780 x1874672085699968/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828883 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828858.376932:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718780 x1874672085699968/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828883 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828858.376943:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828858.376946:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828858.376950:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085699968:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828858.376972:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085700096:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787828858.376994:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828858.377007:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828858/real 1787828858] req@ffff9e35da71a1c0 x1874672085700096/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828883 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828858.377022:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a1c0 x1874672085700096/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828883 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828858.377031:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828858.377035:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828858.377038:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085700096:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828858.377045:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085700224:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787828858.377065:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828858.377079:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828858/real 1787828858] req@ffff9e35da7183c0 x1874672085700224/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828883 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828858.377092:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672085700224/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828883 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828858.377102:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828858.377106:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828858.377111:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085700224:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828858.377119:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085700352:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828858.377140:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828858.377152:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828858/real 1787828858] req@ffff9e35da71a940 x1874672085700352/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828883 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828858.377166:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672085700352/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828883 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828858.377176:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828858.377180:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828858.377184:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085700352:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828858.377191:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085700480:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828858.377228:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828858.377240:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828858/real 1787828858] req@ffff9e35da718f00 x1874672085700480/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828883 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828858.377253:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672085700480/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828883 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828858.377263:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828858.377267:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828858.377270:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085700480:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828858.503871:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787828858.503872:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828858.503872:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828858.503944:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828859.527924:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828859.527950:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828859.527957:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828859.528001:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828860.551946:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787828860.552028:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787828860.552051:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828860.552055:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828861.575941:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828861.575965:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787828861.575967:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787828861.576029:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828862.599865:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828862.599878:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:3.0:1787828862.599994:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828862.600036:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828863.495941:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828863.495953:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787828863.495958:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828863.495970:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2298 00000100:00080000:3.0:1787828863.495982:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828863.495991:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787828863.496031:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a1c0 x1874672085700608/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828863.496067:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828863.496073:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828863.496075:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828863.496080:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2298 00000100:00080000:3.0:1787828863.496084:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828863.496089:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828863.496100:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828863.496117:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498000 x1874672085700736/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828863.496131:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085700608:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1787828863.496133:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828863.496143:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787828863.496145:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828863.496151:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2298 00000100:00080000:3.0:1787828863.496156:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828863.496160:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828863.496166:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828863.496198:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c64992c0 x1874672085700864/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828863.496221:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828863.496225:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787828863.496230:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1787828863.496231:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1787828863.496242:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2298 00000100:00080000:3.0:1787828863.496248:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828863.496264:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828863.496271:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:1.0:1787828863.496291:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828863/real 1787828863] req@ffff9e35c649a1c0 x1874672085700608/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828888 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828863.496297:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499e00 x1874672085700992/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828863.496324:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787828863.496331:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a1c0 x1874672085700608/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828888 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828863.496331:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787828863.496336:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828863.496348:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2298 00000100:00080000:1.0:1787828863.496352:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828863.496354:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828863.496362:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:3.0:1787828863.496364:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787828863.496372:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085700608:192.168.203.141@tcp:250:kworker.0 00000008:00100000:3.0:1787828863.496374:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:3.0:1787828863.496395:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c64983c0 x1874672085701120/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828863.496404:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085700736:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1787828863.496423:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828863.496429:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828863.496432:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828863.496438:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2298 00000100:00080000:3.0:1787828863.496443:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1787828863.496449:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1787828863.496451:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828863.496460:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828863.496471:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828863/real 1787828863] req@ffff9e35c6498000 x1874672085700736/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828888 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828863.496477:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b480 x1874672085701248/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828863.496499:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787828863.496500:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498000 x1874672085700736/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828888 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828863.496508:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787828863.496514:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828863.496525:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2298 00000100:00080000:1.0:1787828863.496526:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828863.496530:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828863.496536:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:3.0:1787828863.496540:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787828863.496546:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085700736:192.168.203.141@tcp:38:kworker.0 00000008:00100000:3.0:1787828863.496550:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828863.496559:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c64992c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085700864:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787828863.496568:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b0c0 x1874672085701376/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828863.496581:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828863.496587:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000100:1.0:1787828863.496588:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1787828863.496591:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828863.496609:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2298 00000100:00100000:1.0:1787828863.496613:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828863/real 1787828863] req@ffff9e35c64992c0 x1874672085700864/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828888 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828863.496616:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828863.496637:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787828863.496638:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c64992c0 x1874672085700864/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828888 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1787828863.496648:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00080000:1.0:1787828863.496654:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828863.496670:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828863.496677:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a580 x1874672085701504/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828863.496678:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c64992c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085700864:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828863.496693:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085700992:192.168.203.141@tcp:8:kworker.0 00000100:00080000:3.0:1787828863.496699:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828863.496704:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787828863.496706:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828863.496712:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2298 00000100:00080000:3.0:1787828863.496718:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1787828863.496724:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1787828863.496726:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828863.496741:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:3.0:1787828863.496757:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a1c0 x1874672085701632/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828863.496762:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828863/real 1787828863] req@ffff9e35c6499e00 x1874672085700992/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828888 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828863.496790:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499e00 x1874672085700992/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828888 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828863.496844:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828863.496848:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828863.496862:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085700992:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828863.496889:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c64983c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085701120:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828863.496912:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828863.496924:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828863/real 1787828863] req@ffff9e35c64983c0 x1874672085701120/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828888 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828863.496949:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c64983c0 x1874672085701120/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828888 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828863.496958:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828863.496962:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828863.496965:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c64983c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085701120:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828863.496972:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085701248:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787828863.496990:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828863.497003:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828863/real 1787828863] req@ffff9e35c649b480 x1874672085701248/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828888 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828863.497016:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b480 x1874672085701248/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828888 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828863.497026:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828863.497030:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828863.497034:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085701248:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828863.497041:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085701376:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787828863.497064:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828863.497079:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828863/real 1787828863] req@ffff9e35c649b0c0 x1874672085701376/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828888 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828863.497099:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b0c0 x1874672085701376/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828888 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828863.497105:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828863.497108:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828863.497110:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085701376:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828863.497114:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085701504:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828863.497129:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828863.497136:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828863/real 1787828863] req@ffff9e35c649a580 x1874672085701504/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828888 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828863.497144:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a580 x1874672085701504/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828888 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828863.497149:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828863.497151:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828863.497153:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085701504:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828863.497157:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085701632:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828863.497168:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828863.497194:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828863/real 1787828863] req@ffff9e35c649a1c0 x1874672085701632/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828888 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828863.497203:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a1c0 x1874672085701632/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828888 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828863.497210:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828863.497212:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828863.497214:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085701632:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828863.623950:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787828863.623950:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828863.623951:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828863.624040:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828864.647891:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828864.647891:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787828864.647912:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:0.0:1787828864.647982:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828865.671946:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:0.0:1787828865.671999:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828865.672119:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828865.672300:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787828866.695970:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787828866.695971:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828866.695971:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828866.696106:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828867.719980:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:0.0:1787828867.720088:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828867.720253:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787828867.720254:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787828868.615923:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828868.615931:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828868.615934:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828868.615943:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2303 00000100:00080000:1.0:1787828868.615952:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828868.615958:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787828868.615990:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f70c0 x1874672085701760/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828868.616018:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828868.616021:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828868.616022:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828868.616025:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2303 00000100:00080000:1.0:1787828868.616027:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828868.616029:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828868.616037:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828868.616045:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6940 x1874672085701888/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828868.616056:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828868.616058:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787828868.616059:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828868.616062:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2303 00000100:00080000:1.0:1787828868.616063:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828868.616065:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828868.616068:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828868.616075:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6580 x1874672085702016/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828868.616081:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828868.616083:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828868.616084:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828868.616087:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2303 00000100:00080000:1.0:1787828868.616089:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828868.616091:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828868.616094:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:2.0:1787828868.616099:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085701760:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1787828868.616103:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f43c0 x1874672085702144/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828868.616112:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828868.616114:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828868.616115:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828868.616118:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2303 00000100:00080000:1.0:1787828868.616120:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828868.616123:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828868.616126:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:1.0:1787828868.616148:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f61c0 x1874672085702272/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828868.616155:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828868.616158:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828868.616159:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828868.616162:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2303 00000100:00080000:1.0:1787828868.616171:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828868.616174:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828868.616177:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1787828868.616185:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828868.616189:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4b40 x1874672085702400/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828868.616195:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828868.616197:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787828868.616200:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828868.616203:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2303 00000100:00080000:1.0:1787828868.616204:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828868.616206:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828868.616209:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828868.616216:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7480 x1874672085702528/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828868.616228:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828868.616230:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828868.616238:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828868.616241:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2303 00000100:00080000:1.0:1787828868.616245:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787828868.616246:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828868/real 1787828868] req@ffff9e35da6f70c0 x1874672085701760/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828893 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828868.616250:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828868.616258:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:1.0:1787828868.616269:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5e00 x1874672085702656/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828868.616272:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f70c0 x1874672085701760/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828893 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828868.616296:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828868.616297:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828868.616302:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787828868.616303:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:1.0:1787828868.616309:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787828868.616310:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085701760:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787828868.616320:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2303 00000100:00100000:2.0:1787828868.616323:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085701888:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787828868.616324:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828868.616331:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828868.616335:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:1.0:1787828868.616345:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4000 x1874672085702784/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787828868.616353:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828868.616366:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828868/real 1787828868] req@ffff9e35da6f6940 x1874672085701888/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828893 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828868.616376:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6940 x1874672085701888/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828893 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828868.616383:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828868.616385:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828868.616387:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085701888:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828868.616391:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085702016:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828868.616402:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828868.616408:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828868/real 1787828868] req@ffff9e35da6f6580 x1874672085702016/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828893 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828868.616415:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6580 x1874672085702016/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828893 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828868.616421:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828868.616422:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828868.616424:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085702016:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828868.616449:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085702144:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828868.616460:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828868.616466:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828868/real 1787828868] req@ffff9e35da6f43c0 x1874672085702144/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828893 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828868.616491:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f43c0 x1874672085702144/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828893 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828868.616497:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828868.616499:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828868.616501:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085702144:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828868.616505:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085702272:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828868.616516:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828868.616522:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828868/real 1787828868] req@ffff9e35da6f61c0 x1874672085702272/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828893 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828868.616530:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f61c0 x1874672085702272/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828893 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828868.616535:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828868.616537:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828868.616539:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085702272:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828868.616542:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085702400:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828868.616552:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828868.616558:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828868/real 1787828868] req@ffff9e35da6f4b40 x1874672085702400/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828893 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828868.616565:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4b40 x1874672085702400/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828893 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828868.616575:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828868.616578:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828868.616580:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085702400:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828868.616583:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085702528:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828868.616593:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828868.616599:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828868/real 1787828868] req@ffff9e35da6f7480 x1874672085702528/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828893 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828868.616606:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7480 x1874672085702528/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828893 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828868.616611:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828868.616613:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828868.616615:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085702528:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828868.616618:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085702656:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828868.616646:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828868.616651:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828868/real 1787828868] req@ffff9e35da6f5e00 x1874672085702656/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828893 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828868.616658:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5e00 x1874672085702656/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828893 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828868.616663:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828868.616665:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828868.616668:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085702656:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828868.616672:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085702784:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828868.616681:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828868.616687:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828868/real 1787828868] req@ffff9e35da6f4000 x1874672085702784/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828893 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828868.616693:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4000 x1874672085702784/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828893 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828868.616699:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828868.616700:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828868.616702:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085702784:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828868.743934:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787828868.743946:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787828868.743968:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828868.743999:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828869.767908:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787828869.767945:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828869.767969:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828869.767970:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787828870.791934:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828870.791953:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787828870.792018:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828870.792049:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828871.815877:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:3.0:1787828871.815884:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828871.815903:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828871.815948:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828872.839923:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828872.839923:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828872.839966:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828872.839970:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828873.735967:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828873.735975:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828873.735979:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828873.735987:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2309 00000100:00080000:0.0:1787828873.735996:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828873.736002:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787828873.736035:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718b40 x1874672085702912/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828873.736060:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828873.736063:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828873.736064:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828873.736067:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2309 00000100:00080000:0.0:1787828873.736070:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828873.736072:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828873.736080:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828873.736092:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718f00 x1874672085703040/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828873.736118:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787828873.736118:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085702912:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1787828873.736124:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787828873.736144:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828873.736150:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2309 00000100:00080000:0.0:1787828873.736154:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828873.736157:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828873.736163:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828873.736178:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a940 x1874672085703168/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828873.736188:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828873.736191:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787828873.736211:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828873.736215:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2309 00000100:00080000:0.0:1787828873.736220:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828873.736223:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828873.736229:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000800:00000100:2.0:1787828873.736232:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828873.736247:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7183c0 x1874672085703296/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828873.736259:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828873.736279:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787828873.736280:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828873.736283:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2309 00000100:00080000:0.0:1787828873.736286:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828873.736289:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828873.736292:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:2.0:1787828873.736298:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828873/real 1787828873] req@ffff9e35da718b40 x1874672085702912/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828898 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828873.736303:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a1c0 x1874672085703424/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828873.736317:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828873.736320:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828873.736321:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787828873.736325:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718b40 x1874672085702912/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828898 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828873.736328:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2309 00000100:00080000:0.0:1787828873.736334:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828873.736339:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828873.736345:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1787828873.736346:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828873.736352:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828873.736356:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718780 x1874672085703552/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828873.736360:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085702912:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1787828873.736371:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828873.736392:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787828873.736394:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828873.736399:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2309 00000100:00080000:0.0:1787828873.736402:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828873.736405:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828873.736409:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828873.736415:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085703040:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787828873.736422:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da719a40 x1874672085703680/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828873.736441:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1787828873.736446:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787828873.736454:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787828873.736457:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787828873.736462:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828873/real 1787828873] req@ffff9e35da718f00 x1874672085703040/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828898 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828873.736464:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2309 00000100:00080000:0.0:1787828873.736471:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828873.736480:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828873.736488:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:2.0:1787828873.736492:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672085703040/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828898 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828873.736520:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828873.736530:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828873.736532:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a580 x1874672085703808/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828873.736538:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085703040:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787828873.736559:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828873.736566:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:2.0:1787828873.736567:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085703168:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787828873.736568:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828873.736593:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2309 00000100:00080000:0.0:1787828873.736595:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828873.736598:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828873.736602:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:0.0:1787828873.736613:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718000 x1874672085703936/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787828873.736621:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828873.736638:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828873/real 1787828873] req@ffff9e35da71a940 x1874672085703168/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828898 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828873.736649:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672085703168/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828898 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828873.736656:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828873.736658:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828873.736660:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085703168:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828873.736665:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085703296:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828873.736680:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828873.736687:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828873/real 1787828873] req@ffff9e35da7183c0 x1874672085703296/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828898 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828873.736695:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672085703296/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828898 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828873.736701:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828873.736703:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828873.736705:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085703296:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828873.736708:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085703424:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828873.736718:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828873.736723:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828873/real 1787828873] req@ffff9e35da71a1c0 x1874672085703424/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828898 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828873.736730:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a1c0 x1874672085703424/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828898 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828873.736737:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828873.736738:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828873.736740:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085703424:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828873.736744:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085703552:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828873.736754:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828873.736760:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828873/real 1787828873] req@ffff9e35da718780 x1874672085703552/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828898 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828873.736772:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718780 x1874672085703552/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828898 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828873.736777:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828873.736779:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828873.736781:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085703552:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828873.736796:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085703680:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828873.736806:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828873.736813:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828873/real 1787828873] req@ffff9e35da719a40 x1874672085703680/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828898 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828873.736820:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672085703680/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828898 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828873.736826:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828873.736828:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828873.736829:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085703680:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828873.736833:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085703808:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828873.736842:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828873.736847:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828873/real 1787828873] req@ffff9e35da71a580 x1874672085703808/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828898 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828873.736854:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672085703808/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828898 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828873.736863:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828873.736865:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828873.736867:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085703808:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828873.736870:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085703936:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828873.736879:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828873.736885:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828873/real 1787828873] req@ffff9e35da718000 x1874672085703936/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828898 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828873.736923:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718000 x1874672085703936/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828898 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828873.736929:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828873.736931:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828873.736933:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085703936:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828873.863935:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787828873.863941:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828873.863953:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:0.0:1787828873.864004:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828874.887963:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787828874.887973:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828874.888017:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828874.888034:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828875.911900:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828875.911900:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828875.911956:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828875.911956:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828876.935872:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828876.935898:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787828876.935902:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787828876.935924:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828877.959892:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787828877.959908:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828877.959912:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828877.960003:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828878.855857:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828878.855866:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828878.855869:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828878.855877:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2314 00000100:00080000:0.0:1787828878.855887:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828878.855895:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787828878.855931:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7192c0 x1874672085704064/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828878.855955:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828878.855958:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828878.855960:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828878.855962:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2314 00000100:00080000:0.0:1787828878.855965:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828878.855968:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828878.855976:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828878.855988:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718000 x1874672085704192/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828878.855995:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828878.855997:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787828878.855999:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828878.856001:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2314 00000100:00080000:0.0:1787828878.856003:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828878.856006:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828878.856010:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828878.856018:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a580 x1874672085704320/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828878.856028:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787828878.856028:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085704064:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1787828878.856036:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787828878.856039:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828878.856043:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2314 00000100:00080000:0.0:1787828878.856045:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828878.856047:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828878.856051:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:0.0:1787828878.856059:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da719a40 x1874672085704448/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828878.856079:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828878.856082:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787828878.856084:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828878.856087:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2314 00000100:00080000:0.0:1787828878.856090:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828878.856094:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828878.856116:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000800:00000100:2.0:1787828878.856132:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828878.856142:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718780 x1874672085704576/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828878.856158:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828878.856162:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828878.856165:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828878.856187:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2314 00000100:00080000:0.0:1787828878.856192:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828878.856199:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787828878.856201:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828878/real 1787828878] req@ffff9e35da7192c0 x1874672085704064/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828903 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1787828878.856205:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828878.856225:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a1c0 x1874672085704704/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828878.856232:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7192c0 x1874672085704064/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828903 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828878.856258:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828878.856270:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787828878.856270:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828878.856277:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828878.856279:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828878.856290:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085704064:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1787828878.856295:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2314 00000100:00100000:2.0:1787828878.856304:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085704192:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787828878.856305:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828878.856320:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828878.856331:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1787828878.856344:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828878.856354:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7183c0 x1874672085704832/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828878.856363:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828878/real 1787828878] req@ffff9e35da718000 x1874672085704192/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828903 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828878.856371:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828878.856383:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787828878.856388:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828878.856397:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2314 00000100:00100000:2.0:1787828878.856401:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718000 x1874672085704192/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828903 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828878.856404:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828878.856417:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:2.0:1787828878.856420:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000008:00100000:0.0:1787828878.856424:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00080000:2.0:1787828878.856429:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828878.856436:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085704192:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787828878.856444:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a940 x1874672085704960/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828878.856456:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828878.856460:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787828878.856461:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828878.856465:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2314 00000100:00080000:0.0:1787828878.856469:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828878.856472:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828878.856479:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:2.0:1787828878.856479:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085704320:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787828878.856496:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718f00 x1874672085705088/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787828878.856526:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828878.856560:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828878/real 1787828878] req@ffff9e35da71a580 x1874672085704320/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828903 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828878.856581:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672085704320/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828903 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828878.856587:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828878.856589:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828878.856591:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085704320:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828878.856597:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085704448:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828878.856612:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828878.856620:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828878/real 1787828878] req@ffff9e35da719a40 x1874672085704448/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828903 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828878.856628:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672085704448/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828903 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828878.856634:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828878.856636:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828878.856638:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085704448:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828878.856642:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085704576:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828878.856651:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828878.856657:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828878/real 1787828878] req@ffff9e35da718780 x1874672085704576/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828903 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828878.856664:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718780 x1874672085704576/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828903 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828878.856670:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828878.856671:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828878.856673:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085704576:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828878.856677:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085704704:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828878.856686:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828878.856693:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828878/real 1787828878] req@ffff9e35da71a1c0 x1874672085704704/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828903 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828878.856701:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a1c0 x1874672085704704/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828903 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828878.856707:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828878.856710:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828878.856712:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085704704:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828878.856716:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085704832:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828878.856726:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828878.856732:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828878/real 1787828878] req@ffff9e35da7183c0 x1874672085704832/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828903 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828878.856741:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672085704832/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828903 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828878.856748:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828878.856750:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828878.856753:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085704832:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828878.856756:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085704960:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828878.856766:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828878.856772:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828878/real 1787828878] req@ffff9e35da71a940 x1874672085704960/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828903 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828878.856779:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672085704960/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828903 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828878.856789:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828878.856790:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828878.856792:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085704960:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828878.856830:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085705088:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828878.856843:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828878.856850:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828878/real 1787828878] req@ffff9e35da718f00 x1874672085705088/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828903 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828878.856857:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672085705088/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828903 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828878.856863:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828878.856865:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828878.856867:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085705088:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828878.983927:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787828878.983929:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828878.983979:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828878.983993:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828880.007880:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828880.007880:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828880.007887:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:0.0:1787828880.007928:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828881.031912:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828881.031914:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828881.031922:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828881.031979:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828882.055868:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828882.055889:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828882.055906:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828882.055922:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828883.079906:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828883.079909:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828883.079945:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:0.0:1787828883.079974:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828883.975920:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828883.975927:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828883.975931:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828883.975950:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2319 00000100:00080000:2.0:1787828883.975961:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828883.975967:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787828883.976000:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c780 x1874672085705216/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828883.976024:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828883.976040:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828883.976043:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828883.976047:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2319 00000100:00080000:2.0:1787828883.976050:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828883.976067:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828883.976078:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828883.976083:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085705216:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1787828883.976097:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c3c0 x1874672085705344/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828883.976111:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828883.976115:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787828883.976117:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828883.976122:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2319 00000100:00080000:2.0:1787828883.976126:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828883.976131:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828883.976136:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1787828883.976140:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828883.976151:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498ed00 x1874672085705472/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828883.976159:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828883.976161:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787828883.976163:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828883.976166:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2319 00000100:00080000:2.0:1787828883.976169:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828883.976172:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828883.976177:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:1.0:1787828883.976178:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828883/real 1787828883] req@ffff9e35c498c780 x1874672085705216/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828908 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828883.976187:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498de00 x1874672085705600/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828883.976195:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787828883.976198:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c780 x1874672085705216/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828908 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828883.976199:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787828883.976203:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828883.976210:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2319 00000100:00080000:2.0:1787828883.976214:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828883.976215:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828883.976218:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828883.976222:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00080000:1.0:1787828883.976223:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828883.976229:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085705216:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1787828883.976234:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498fc00 x1874672085705728/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828883.976242:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828883.976244:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828883.976246:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828883.976248:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2319 00000100:00080000:2.0:1787828883.976250:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1787828883.976255:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085705344:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1787828883.976255:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828883.976261:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828883.976271:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e940 x1874672085705856/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828883.976280:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1787828883.976281:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1787828883.976285:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787828883.976287:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828883.976292:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2319 00000100:00080000:2.0:1787828883.976296:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1787828883.976299:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828883/real 1787828883] req@ffff9e35c498c3c0 x1874672085705344/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828908 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828883.976300:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828883.976306:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828883.976319:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c780 x1874672085705984/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828883.976320:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c3c0 x1874672085705344/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828908 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828883.976344:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828883.976354:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828883.976359:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828883.976359:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828883.976367:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:2.0:1787828883.976370:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2319 00000100:00100000:1.0:1787828883.976373:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085705344:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1787828883.976377:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1787828883.976381:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085705472:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1787828883.976387:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828883.976392:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000800:00000100:1.0:1787828883.976401:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828883.976405:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498d680 x1874672085706112/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828883.976413:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828883/real 1787828883] req@ffff9e35c498ed00 x1874672085705472/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828908 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828883.976415:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828883.976422:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787828883.976426:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828883.976433:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2319 00000100:00100000:1.0:1787828883.976436:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498ed00 x1874672085705472/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828908 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828883.976438:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828883.976445:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828883.976452:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00080000:1.0:1787828883.976453:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828883.976459:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828883.976462:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cf00 x1874672085706240/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828883.976465:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085705472:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828883.976479:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085705600:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828883.976504:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828883.976516:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828883/real 1787828883] req@ffff9e35c498de00 x1874672085705600/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828908 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828883.976530:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498de00 x1874672085705600/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828908 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828883.976538:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828883.976540:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828883.976543:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085705600:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828883.976548:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085705728:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828883.976571:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828883.976578:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828883/real 1787828883] req@ffff9e35c498fc00 x1874672085705728/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828908 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828883.976586:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498fc00 x1874672085705728/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828908 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828883.976593:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828883.976595:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828883.976597:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085705728:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828883.976619:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085705856:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787828883.976631:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828883.976638:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828883/real 1787828883] req@ffff9e35c498e940 x1874672085705856/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828908 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828883.976646:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e940 x1874672085705856/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828908 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828883.976652:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828883.976654:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828883.976656:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085705856:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828883.976660:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085705984:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787828883.976669:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828883.976675:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828883/real 1787828883] req@ffff9e35c498c780 x1874672085705984/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828908 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828883.976682:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c780 x1874672085705984/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828908 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828883.976688:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828883.976690:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828883.976691:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085705984:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828883.976695:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085706112:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828883.976703:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828883.976708:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828883/real 1787828883] req@ffff9e35c498d680 x1874672085706112/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828908 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828883.976715:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498d680 x1874672085706112/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828908 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828883.976726:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828883.976729:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828883.976731:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085706112:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828883.976735:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085706240:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828883.976745:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828883.976751:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828883/real 1787828883] req@ffff9e35c498cf00 x1874672085706240/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828908 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828883.976759:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498cf00 x1874672085706240/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828908 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828883.976765:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828883.976767:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828883.976769:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085706240:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787828884.103872:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828884.103882:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828884.103910:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828884.104046:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828885.127864:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828885.127882:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787828885.127915:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828885.127938:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828886.151912:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828886.151934:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828886.151964:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787828886.151972:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828887.175888:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828887.175897:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787828887.175899:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828887.175945:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828888.199944:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787828888.199955:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787828888.199971:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828888.200112:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828889.096012:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828889.096042:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787828889.096049:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828889.096062:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2324 00000100:00080000:3.0:1787828889.096078:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828889.096088:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787828889.096146:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649ad00 x1874672085706368/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828889.096181:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828889.096186:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828889.096189:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828889.096194:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2324 00000100:00080000:3.0:1787828889.096199:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828889.096204:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828889.096219:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828889.096231:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085706368:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1787828889.096244:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499a40 x1874672085706496/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828889.096263:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828889.096267:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787828889.096270:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828889.096276:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2324 00000100:00080000:3.0:1787828889.096299:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828889.096305:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828889.096312:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1787828889.096313:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828889.096336:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498780 x1874672085706624/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828889.096369:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828889/real 1787828889] req@ffff9e35c649ad00 x1874672085706368/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828914 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828889.096386:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649ad00 x1874672085706368/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828914 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828889.096395:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828889.096398:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828889.096402:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085706368:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1787828889.096423:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085706496:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828889.096442:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828889.096451:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828889/real 1787828889] req@ffff9e35c6499a40 x1874672085706496/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828914 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828889.096467:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499a40 x1874672085706496/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828914 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828889.096496:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828889.096497:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828889.096505:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828889.096508:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085706496:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828889.096518:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085706624:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828889.096531:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828889.096538:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828889/real 1787828889] req@ffff9e35c6498780 x1874672085706624/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828914 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828889.096546:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498780 x1874672085706624/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828914 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828889.096553:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828889.096555:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828889.096557:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085706624:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1787828889.096693:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787828889.096697:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828889.096704:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2324 00000100:00080000:3.0:1787828889.096720:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828889.096727:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828889.096736:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:3.0:1787828889.096862:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a940 x1874672085706752/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828889.096896:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828889.096901:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787828889.096904:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828889.096910:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2324 00000100:00080000:3.0:1787828889.096916:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828889.096922:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787828889.096929:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085706752:192.168.203.141@tcp:8:kworker.0 00000008:00100000:3.0:1787828889.096929:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:3.0:1787828889.096964:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498b40 x1874672085706880/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787828889.096977:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1787828889.096978:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828889.096984:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828889.096986:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787828889.097009:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828889/real 1787828889] req@ffff9e35c649a940 x1874672085706752/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828914 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828889.097015:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2324 00000100:00080000:3.0:1787828889.097031:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828889.097038:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787828889.097042:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a940 x1874672085706752/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828914 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1787828889.097049:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1787828889.097071:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828889.097076:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828889.097077:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499680 x1874672085707008/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828889.097089:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085706752:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828889.097129:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085706880:192.168.203.141@tcp:8:kworker.0 00000100:00080000:3.0:1787828889.097131:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828889.097147:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787828889.097151:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828889.097157:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2324 00000100:00080000:3.0:1787828889.097165:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828889.097171:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:2.0:1787828889.097178:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:3.0:1787828889.097183:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828889.097196:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828889/real 1787828889] req@ffff9e35c6498b40 x1874672085706880/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828914 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828889.097204:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498f00 x1874672085707136/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828889.097251:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498b40 x1874672085706880/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828914 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828889.097254:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828889.097270:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787828889.097277:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828889.097282:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828889.097288:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2324 00000100:00080000:2.0:1787828889.097295:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:3.0:1787828889.097296:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787828889.097304:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085706880:192.168.203.141@tcp:8:kworker.0 00000100:00080000:3.0:1787828889.097306:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828889.097324:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:2.0:1787828889.097335:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085707008:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787828889.097345:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b840 x1874672085707264/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787828889.097360:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828889.097372:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828889/real 1787828889] req@ffff9e35c6499680 x1874672085707008/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828914 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828889.097380:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787828889.097386:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499680 x1874672085707008/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828914 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828889.097392:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787828889.097405:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828889.097409:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828889.097416:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2324 00000100:00080000:2.0:1787828889.097417:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:3.0:1787828889.097432:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787828889.097433:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085707008:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1787828889.097449:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787828889.097451:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085707136:192.168.203.141@tcp:38:kworker.0 00000008:00100000:3.0:1787828889.097464:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000800:00000100:2.0:1787828889.097478:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828889.097486:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649bc00 x1874672085707392/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828889.097494:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828889/real 1787828889] req@ffff9e35c6498f00 x1874672085707136/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828914 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828889.097544:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498f00 x1874672085707136/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828914 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828889.097550:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828889.097555:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828889.097561:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085707136:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828889.097578:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085707264:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828889.097591:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828889.097598:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828889/real 1787828889] req@ffff9e35c649b840 x1874672085707264/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828914 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828889.097605:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b840 x1874672085707264/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828914 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828889.097610:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828889.097612:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828889.097614:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085707264:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828889.097618:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085707392:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828889.097639:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828889.097646:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828889/real 1787828889] req@ffff9e35c649bc00 x1874672085707392/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828914 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828889.097653:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649bc00 x1874672085707392/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828914 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828889.097659:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828889.097660:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828889.097662:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085707392:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787828889.223917:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828889.223944:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787828889.223978:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:0.0:1787828889.224042:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828890.247906:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828890.247914:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828890.247946:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787828890.247951:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828891.271934:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828891.272033:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787828891.272037:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787828891.272109:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828892.295967:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828892.296043:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:0.0:1787828892.296048:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828892.296086:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828893.319910:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828893.319910:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787828893.319948:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828893.320020:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828894.216020:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828894.216034:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787828894.216039:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828894.216048:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2329 00000100:00080000:3.0:1787828894.216071:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828894.216078:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787828894.216137:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649bc00 x1874672085707520/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828894.216169:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828894.216172:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828894.216179:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828894.216183:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2329 00000100:00080000:3.0:1787828894.216186:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828894.216189:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828894.216215:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828894.216246:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b840 x1874672085707648/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828894.216260:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085707520:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1787828894.216286:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828894.216292:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787828894.216295:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828894.216304:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2329 00000100:00080000:3.0:1787828894.216321:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828894.216327:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828894.216333:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1787828894.216355:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828894.216361:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498f00 x1874672085707776/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828894.216372:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828894.216376:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787828894.216378:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828894.216382:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2329 00000100:00080000:3.0:1787828894.216386:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828894.216389:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828894.216394:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:3.0:1787828894.216406:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499680 x1874672085707904/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828894.216424:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828894.216445:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787828894.216450:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787828894.216452:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828894/real 1787828894] req@ffff9e35c649bc00 x1874672085707520/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828919 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828894.216459:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2329 00000100:00080000:3.0:1787828894.216468:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828894.216478:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828894.216488:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:2.0:1787828894.216500:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649bc00 x1874672085707520/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828919 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828894.216526:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498b40 x1874672085708032/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828894.216550:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828894.216557:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828894.216560:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:3.0:1787828894.216568:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:2.0:1787828894.216573:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085707520:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1787828894.216592:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828894.216599:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2329 00000100:00080000:3.0:1787828894.216607:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828894.216614:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828894.216620:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828894.216636:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a940 x1874672085708160/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828894.216642:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085707648:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1787828894.216683:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828894.216692:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787828894.216694:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828894.216701:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2329 00000800:00000100:2.0:1787828894.216702:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1787828894.216708:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828894.216716:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787828894.216723:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828894/real 1787828894] req@ffff9e35c649b840 x1874672085707648/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828919 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1787828894.216724:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828894.216759:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498780 x1874672085708288/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828894.216761:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b840 x1874672085707648/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828919 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828894.216837:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828894.216840:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787828894.216842:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828894.216848:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2329 00000100:00080000:2.0:1787828894.216851:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828894.216852:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828894.216863:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:3.0:1787828894.216864:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828894.216882:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:2.0:1787828894.216884:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085707648:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787828894.216902:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499a40 x1874672085708416/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828894.216903:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085707776:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1787828894.216922:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828894.216927:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787828894.216930:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828894.216937:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2329 00000800:00000100:2.0:1787828894.216943:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1787828894.216946:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828894.216968:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828894.216977:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:2.0:1787828894.216978:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828894/real 1787828894] req@ffff9e35c6498f00 x1874672085707776/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828919 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828894.217015:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649ad00 x1874672085708544/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828894.217016:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498f00 x1874672085707776/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828919 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828894.217054:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828894.217060:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828894.217075:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085707776:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828894.217082:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085707904:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828894.217124:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828894.217137:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828894/real 1787828894] req@ffff9e35c6499680 x1874672085707904/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828919 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828894.217155:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499680 x1874672085707904/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828919 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828894.217169:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828894.217174:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828894.217188:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085707904:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828894.217198:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085708032:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828894.217231:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828894.217259:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828894/real 1787828894] req@ffff9e35c6498b40 x1874672085708032/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828919 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828894.217278:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498b40 x1874672085708032/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828919 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828894.217291:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828894.217296:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828894.217302:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085708032:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828894.217334:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085708160:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828894.217375:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828894.217409:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828894/real 1787828894] req@ffff9e35c649a940 x1874672085708160/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828919 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828894.217439:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a940 x1874672085708160/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828919 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828894.217469:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828894.217474:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828894.217479:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085708160:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828894.217488:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085708288:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828894.217510:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828894.217521:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828894/real 1787828894] req@ffff9e35c6498780 x1874672085708288/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828919 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828894.217531:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498780 x1874672085708288/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828919 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828894.217556:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828894.217558:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828894.217561:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085708288:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828894.217566:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085708416:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828894.217579:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828894.217586:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828894/real 1787828894] req@ffff9e35c6499a40 x1874672085708416/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828919 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828894.217595:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499a40 x1874672085708416/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828919 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828894.217602:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828894.217604:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828894.217607:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085708416:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828894.217611:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085708544:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828894.217622:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828894.217630:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828894/real 1787828894] req@ffff9e35c649ad00 x1874672085708544/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828919 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828894.217638:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649ad00 x1874672085708544/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828919 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828894.217645:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828894.217647:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828894.217649:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085708544:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828894.343862:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787828894.343915:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828894.343972:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828894.343990:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828895.367893:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:3.0:1787828895.367893:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787828895.367966:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787828895.367994:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828896.391957:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:3.0:1787828896.391957:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787828896.391960:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828896.392050:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828897.415955:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787828897.415967:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:3.0:1787828897.415987:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828897.416068:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828898.439998:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828898.439999:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787828898.440024:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787828898.440061:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828899.335994:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828899.336005:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828899.336010:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828899.336024:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2334 00000100:00080000:2.0:1787828899.336036:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828899.336045:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787828899.336096:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f840 x1874672085708672/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828899.336130:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828899.336134:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828899.336136:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828899.336141:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2334 00000100:00080000:2.0:1787828899.336144:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828899.336148:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828899.336161:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828899.336178:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c000 x1874672085708800/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828899.336192:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828899.336199:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:1.0:1787828899.336201:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085708672:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1787828899.336203:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828899.336220:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2334 00000100:00080000:2.0:1787828899.336224:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828899.336227:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828899.336232:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828899.336254:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cb40 x1874672085708928/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828899.336263:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828899.336267:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787828899.336269:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828899.336274:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2334 00000100:00080000:2.0:1787828899.336278:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828899.336282:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828899.336287:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:2.0:1787828899.336297:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e1c0 x1874672085709056/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787828899.336313:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1787828899.336331:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828899.336336:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787828899.336338:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828899.336345:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2334 00000100:00080000:2.0:1787828899.336350:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828899.336356:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828899.336361:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:1.0:1787828899.336374:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828899/real 1787828899] req@ffff9e35c498f840 x1874672085708672/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828924 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828899.336383:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f480 x1874672085709184/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828899.336406:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f840 x1874672085708672/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828924 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828899.336415:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828899.336422:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828899.336424:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828899.336430:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:2.0:1787828899.336433:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787828899.336436:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085708672:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1787828899.336453:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2334 00000100:00080000:2.0:1787828899.336460:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828899.336465:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828899.336473:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828899.336477:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085708800:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828899.336497:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f0c0 x1874672085709312/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787828899.336513:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1787828899.336528:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828899.336532:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787828899.336535:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787828899.336542:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828899/real 1787828899] req@ffff9e35c498c000 x1874672085708800/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828924 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828899.336548:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2334 00000100:00080000:2.0:1787828899.336560:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828899.336575:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787828899.336580:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c000 x1874672085708800/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828924 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1787828899.336588:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00080000:1.0:1787828899.336596:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828899.336601:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828899.336607:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e580 x1874672085709440/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828899.336608:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085708800:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1787828899.336639:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787828899.336643:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085708928:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1787828899.336645:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787828899.336652:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828899.336662:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2334 00000100:00080000:2.0:1787828899.336671:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828899.336677:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:1.0:1787828899.336687:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:2.0:1787828899.336691:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:1.0:1787828899.336701:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828899/real 1787828899] req@ffff9e35c498cb40 x1874672085708928/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828924 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828899.336712:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f840 x1874672085709568/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828899.336725:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498cb40 x1874672085708928/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828924 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828899.336765:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828899.336770:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:2.0:1787828899.336772:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828899.336781:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:1.0:1787828899.336783:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085708928:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1787828899.336791:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787828899.336843:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085709056:192.168.203.141@tcp:8:kworker.0 00000100:00080000:2.0:1787828899.336872:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2334 00000800:00000100:1.0:1787828899.336873:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1787828899.336888:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1787828899.336900:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828899/real 1787828899] req@ffff9e35c498e1c0 x1874672085709056/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828924 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828899.336901:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787828899.336940:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e1c0 x1874672085709056/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828924 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1787828899.336940:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00080000:1.0:1787828899.336962:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828899.336966:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828899.336970:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085709056:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828899.336978:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cf00 x1874672085709696/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828899.336980:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085709184:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828899.337026:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828899.337042:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828899/real 1787828899] req@ffff9e35c498f480 x1874672085709184/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828924 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828899.337052:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f480 x1874672085709184/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828924 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828899.337059:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828899.337061:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828899.337063:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085709184:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828899.337084:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085709312:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787828899.337096:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828899.337103:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828899/real 1787828899] req@ffff9e35c498f0c0 x1874672085709312/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828924 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828899.337116:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f0c0 x1874672085709312/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828924 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828899.337122:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828899.337124:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828899.337126:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085709312:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828899.337130:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085709440:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787828899.337141:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828899.337147:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828899/real 1787828899] req@ffff9e35c498e580 x1874672085709440/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828924 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828899.337165:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e580 x1874672085709440/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828924 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828899.337170:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828899.337172:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828899.337174:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085709440:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828899.337177:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085709568:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828899.337187:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828899.337193:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828899/real 1787828899] req@ffff9e35c498f840 x1874672085709568/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828924 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828899.337200:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f840 x1874672085709568/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828924 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828899.337210:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828899.337213:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828899.337215:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085709568:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828899.337219:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085709696:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828899.337228:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828899.337234:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828899/real 1787828899] req@ffff9e35c498cf00 x1874672085709696/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828924 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828899.337241:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498cf00 x1874672085709696/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828924 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828899.337247:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828899.337249:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828899.337250:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085709696:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787828899.463887:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787828899.463938:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828899.463969:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828899.464040:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828900.487887:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787828900.487927:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787828900.487946:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828900.487967:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828901.511898:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828901.511983:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787828901.511987:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:1.0:1787828901.512070:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828902.535933:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787828902.535972:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828902.536008:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828902.536038:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828903.559875:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828903.559880:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787828903.559906:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787828903.559970:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828904.455974:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828904.455984:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828904.455989:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828904.456001:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2339 00000100:00080000:1.0:1787828904.456014:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828904.456022:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787828904.456067:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7840 x1874672085709824/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828904.456097:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828904.456112:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828904.456114:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828904.456118:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2339 00000100:00080000:1.0:1787828904.456121:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828904.456124:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828904.456134:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828904.456145:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085709824:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1787828904.456146:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5a40 x1874672085709952/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828904.456163:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828904.456177:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787828904.456179:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828904.456182:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2339 00000100:00080000:1.0:1787828904.456185:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828904.456187:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828904.456190:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828904.456200:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4780 x1874672085710080/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828904.456208:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828904.456210:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828904.456212:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828904.456215:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2339 00000100:00080000:1.0:1787828904.456225:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828904.456229:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828904.456243:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000800:00000100:2.0:1787828904.456248:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828904.456255:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6d00 x1874672085710208/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828904.456262:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828904.456265:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828904.456267:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828904.456270:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2339 00000100:00080000:1.0:1787828904.456273:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828904.456275:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828904.456278:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:1.0:1787828904.456285:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4f00 x1874672085710336/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828904.456292:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828904.456295:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828904.456299:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787828904.456299:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828904/real 1787828904] req@ffff9e35da6f7840 x1874672085709824/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828929 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828904.456307:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2339 00000100:00080000:1.0:1787828904.456312:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828904.456317:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828904.456322:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828904.456328:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7840 x1874672085709824/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828929 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828904.456334:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4000 x1874672085710464/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828904.456343:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828904.456345:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828904.456349:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:1.0:1787828904.456350:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787828904.456354:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787828904.456356:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085709824:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787828904.456359:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2339 00000100:00080000:1.0:1787828904.456366:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828904.456368:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828904.456371:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828904.456378:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5e00 x1874672085710592/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828904.456386:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787828904.456387:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085709952:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787828904.456394:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828904.456396:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828904.456401:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2339 00000100:00080000:1.0:1787828904.456404:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828904.456407:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828904.456411:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000800:00000100:2.0:1787828904.456414:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828904.456423:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7480 x1874672085710720/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828904.456429:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828904/real 1787828904] req@ffff9e35da6f5a40 x1874672085709952/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828929 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828904.456461:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787828904.456465:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5a40 x1874672085709952/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828929 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828904.456466:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828904.456470:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828904.456477:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2339 00000100:00080000:2.0:1787828904.456480:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828904.456481:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828904.456487:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828904.456489:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00080000:2.0:1787828904.456494:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828904.456498:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4b40 x1874672085710848/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828904.456502:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085709952:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828904.456513:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085710080:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828904.456532:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828904.456540:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828904/real 1787828904] req@ffff9e35da6f4780 x1874672085710080/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828929 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828904.456549:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4780 x1874672085710080/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828929 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828904.456555:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828904.456558:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828904.456560:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085710080:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828904.456564:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085710208:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828904.456575:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828904.456581:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828904/real 1787828904] req@ffff9e35da6f6d00 x1874672085710208/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828929 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828904.456589:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6d00 x1874672085710208/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828929 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828904.456594:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828904.456596:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828904.456598:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085710208:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828904.456601:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085710336:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828904.456611:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828904.456617:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828904/real 1787828904] req@ffff9e35da6f4f00 x1874672085710336/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828929 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828904.456624:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4f00 x1874672085710336/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828929 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828904.456630:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828904.456632:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828904.456635:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085710336:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828904.456638:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085710464:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828904.456648:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828904.456653:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828904/real 1787828904] req@ffff9e35da6f4000 x1874672085710464/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828929 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828904.456660:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4000 x1874672085710464/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828929 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828904.456666:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828904.456668:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828904.456669:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085710464:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828904.456689:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085710592:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828904.456700:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828904.456714:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828904/real 1787828904] req@ffff9e35da6f5e00 x1874672085710592/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828929 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828904.456722:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5e00 x1874672085710592/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828929 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828904.456727:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828904.456729:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828904.456731:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085710592:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828904.456735:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085710720:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828904.456745:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828904.456750:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828904/real 1787828904] req@ffff9e35da6f7480 x1874672085710720/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828929 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828904.456757:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7480 x1874672085710720/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828929 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828904.456763:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828904.456764:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828904.456766:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085710720:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828904.456770:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085710848:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828904.456779:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828904.456784:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828904/real 1787828904] req@ffff9e35da6f4b40 x1874672085710848/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828929 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828904.456791:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4b40 x1874672085710848/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828929 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828904.456823:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828904.456825:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828904.456827:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085710848:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787828904.583926:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828904.583926:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828904.583926:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787828904.584077:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828905.608006:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828905.608045:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828905.608092:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828905.608102:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828906.631916:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787828906.631921:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828906.631944:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787828906.632022:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828907.655954:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787828907.655974:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828907.656025:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787828907.656061:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828908.679920:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828908.679928:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787828908.679967:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828908.679989:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828909.575918:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828909.575929:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828909.575933:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828909.575944:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2344 00000100:00080000:0.0:1787828909.575955:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828909.575961:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787828909.576006:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718b40 x1874672085710976/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828909.576040:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828909.576044:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828909.576046:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828909.576049:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2344 00000100:00080000:0.0:1787828909.576053:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828909.576058:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828909.576072:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828909.576091:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718f00 x1874672085711104/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828909.576101:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085710976:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1787828909.576110:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828909.576115:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787828909.576117:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828909.576124:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2344 00000100:00080000:0.0:1787828909.576129:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828909.576134:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828909.576141:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828909.576158:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a940 x1874672085711232/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828909.576171:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1787828909.576176:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787828909.576178:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787828909.576183:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828909.576190:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2344 00000100:00080000:0.0:1787828909.576197:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828909.576203:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828909.576210:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:0.0:1787828909.576227:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7183c0 x1874672085711360/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828909.576234:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828909/real 1787828909] req@ffff9e35da718b40 x1874672085710976/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828934 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828909.576243:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828909.576251:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787828909.576253:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828909.576260:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2344 00000100:00080000:0.0:1787828909.576264:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787828909.576265:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718b40 x1874672085710976/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828934 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828909.576273:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828909.576282:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00080000:2.0:1787828909.576287:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828909.576295:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828909.576302:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085710976:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1787828909.576303:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a1c0 x1874672085711488/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828909.576318:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828909.576323:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828909.576325:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787828909.576333:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085711104:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787828909.576334:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2344 00000100:00080000:0.0:1787828909.576343:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828909.576350:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828909.576357:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1787828909.576360:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828909.576374:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828909/real 1787828909] req@ffff9e35da718f00 x1874672085711104/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828934 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828909.576382:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718780 x1874672085711616/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828909.576388:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672085711104/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828934 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828909.576412:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828909.576416:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:0.0:1787828909.576417:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787828909.576420:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085711104:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787828909.576425:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:2.0:1787828909.576429:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085711232:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787828909.576430:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828909.576441:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2344 00000100:00080000:0.0:1787828909.576447:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828909.576452:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:2.0:1787828909.576455:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:0.0:1787828909.576461:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828909.576466:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828909/real 1787828909] req@ffff9e35da71a940 x1874672085711232/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828934 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828909.576478:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672085711232/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828934 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828909.576480:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da719a40 x1874672085711744/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828909.576491:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828909.576498:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828909.576503:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085711232:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787828909.576507:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787828909.576510:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085711360:192.168.203.141@tcp:8:kworker.0 00000100:00080000:0.0:1787828909.576518:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787828909.576522:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1787828909.576530:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787828909.576531:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2344 00000100:00080000:0.0:1787828909.576539:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787828909.576540:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828909/real 1787828909] req@ffff9e35da7183c0 x1874672085711360/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828934 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828909.576549:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787828909.576555:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672085711360/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828934 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1787828909.576559:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00080000:2.0:1787828909.576562:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828909.576566:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828909.576569:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085711360:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828909.576575:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085711488:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787828909.576578:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a580 x1874672085711872/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787828909.576590:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787828909.576592:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828909.576598:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:2.0:1787828909.576599:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828909/real 1787828909] req@ffff9e35da71a1c0 x1874672085711488/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828934 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828909.576602:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828909.576614:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2344 00000100:00080000:0.0:1787828909.576624:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787828909.576627:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a1c0 x1874672085711488/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828934 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828909.576633:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828909.576636:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828909.576640:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085711488:192.168.203.141@tcp:8:kworker.0 00000100:00080000:0.0:1787828909.576648:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828909.576655:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:2.0:1787828909.576660:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085711616:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787828909.576671:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718000 x1874672085712000/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787828909.576689:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828909.576700:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828909/real 1787828909] req@ffff9e35da718780 x1874672085711616/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828934 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828909.576708:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718780 x1874672085711616/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828934 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828909.576715:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828909.576717:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828909.576719:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085711616:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828909.576723:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085711744:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828909.576735:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828909.576740:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828909/real 1787828909] req@ffff9e35da719a40 x1874672085711744/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828934 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828909.576748:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672085711744/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828934 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828909.576758:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828909.576759:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828909.576761:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085711744:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828909.576764:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085711872:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828909.576774:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828909.576780:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828909/real 1787828909] req@ffff9e35da71a580 x1874672085711872/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828934 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828909.576786:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672085711872/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828934 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828909.576792:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828909.576794:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828909.576796:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085711872:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828909.576843:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085712000:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828909.576873:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828909.576882:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828909/real 1787828909] req@ffff9e35da718000 x1874672085712000/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828934 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828909.576890:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718000 x1874672085712000/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828934 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828909.576896:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828909.576898:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828909.576901:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085712000:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828909.703925:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787828909.703961:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:1.0:1787828909.703974:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828909.703975:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787828910.727925:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828910.727925:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828910.727935:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787828910.728024:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828911.751908:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828911.751954:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828911.751967:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828911.751983:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828912.775936:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828912.775954:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787828912.775985:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828912.776002:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828913.799966:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828913.800009:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787828913.800016:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787828913.800095:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828914.695933:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828914.695940:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828914.695943:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828914.695952:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2350 00000100:00080000:1.0:1787828914.695962:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828914.695968:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787828914.696001:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f61c0 x1874672085712128/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828914.696023:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828914.696026:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828914.696027:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828914.696030:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2350 00000100:00080000:1.0:1787828914.696032:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828914.696034:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828914.696042:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828914.696052:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f43c0 x1874672085712256/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828914.696059:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828914.696063:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787828914.696064:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828914.696067:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2350 00000100:00080000:1.0:1787828914.696069:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828914.696071:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828914.696074:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828914.696080:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6580 x1874672085712384/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828914.696088:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828914.696091:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828914.696093:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828914.696102:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2350 00000100:00100000:2.0:1787828914.696105:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085712128:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787828914.696107:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828914.696120:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828914.696128:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:1.0:1787828914.696145:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6940 x1874672085712512/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828914.696154:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828914.696156:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828914.696158:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828914.696161:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2350 00000100:00080000:1.0:1787828914.696164:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828914.696168:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828914.696173:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:1.0:1787828914.696181:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f70c0 x1874672085712640/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828914.696187:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828914.696189:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828914.696191:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828914.696194:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2350 00000100:00080000:1.0:1787828914.696197:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1787828914.696205:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787828914.696209:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828914.696213:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828914.696223:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4b40 x1874672085712768/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828914.696230:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828914.696232:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787828914.696234:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828914.696236:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2350 00000100:00080000:1.0:1787828914.696238:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828914.696241:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828914.696245:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828914.696251:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7480 x1874672085712896/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828914.696258:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828914.696260:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828914.696263:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828914.696268:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2350 00000100:00100000:2.0:1787828914.696269:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828914/real 1787828914] req@ffff9e35da6f61c0 x1874672085712128/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828939 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828914.696271:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828914.696280:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828914.696289:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:1.0:1787828914.696308:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5e00 x1874672085713024/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828914.696315:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f61c0 x1874672085712128/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828939 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828914.696325:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828914.696340:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828914.696347:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828914.696358:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2350 00000100:00080000:2.0:1787828914.696358:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828914.696369:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828914.696375:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:1.0:1787828914.696380:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787828914.696385:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085712128:192.168.203.141@tcp:250:kworker.0 00000008:00100000:1.0:1787828914.696387:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:2.0:1787828914.696400:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085712256:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828914.696406:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4000 x1874672085713152/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787828914.696430:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828914.696458:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828914/real 1787828914] req@ffff9e35da6f43c0 x1874672085712256/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828939 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828914.696472:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f43c0 x1874672085712256/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828939 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828914.696483:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828914.696495:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828914.696499:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085712256:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828914.696530:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085712384:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828914.696548:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828914.696564:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828914/real 1787828914] req@ffff9e35da6f6580 x1874672085712384/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828939 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828914.696573:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6580 x1874672085712384/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828939 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828914.696579:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828914.696582:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828914.696584:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085712384:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828914.696588:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085712512:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828914.696600:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828914.696607:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828914/real 1787828914] req@ffff9e35da6f6940 x1874672085712512/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828939 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828914.696621:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6940 x1874672085712512/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828939 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828914.696627:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828914.696630:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828914.696632:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085712512:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828914.696637:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085712640:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828914.696649:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828914.696656:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828914/real 1787828914] req@ffff9e35da6f70c0 x1874672085712640/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828939 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828914.696664:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f70c0 x1874672085712640/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828939 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828914.696670:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828914.696672:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828914.696674:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085712640:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828914.696677:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085712768:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828914.696688:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828914.696694:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828914/real 1787828914] req@ffff9e35da6f4b40 x1874672085712768/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828939 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828914.696701:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4b40 x1874672085712768/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828939 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828914.696707:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828914.696709:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828914.696710:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085712768:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828914.696714:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085712896:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828914.696724:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828914.696730:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828914/real 1787828914] req@ffff9e35da6f7480 x1874672085712896/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828939 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828914.696742:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7480 x1874672085712896/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828939 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828914.696748:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828914.696749:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828914.696751:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085712896:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828914.696755:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085713024:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828914.696765:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828914.696770:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828914/real 1787828914] req@ffff9e35da6f5e00 x1874672085713024/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828939 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828914.696777:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5e00 x1874672085713024/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828939 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828914.696782:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828914.696784:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828914.696786:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085713024:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828914.696790:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085713152:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828914.696867:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828914.696876:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828914/real 1787828914] req@ffff9e35da6f4000 x1874672085713152/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828939 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828914.696884:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4000 x1874672085713152/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828939 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828914.696894:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828914.696896:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828914.696898:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085713152:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828914.823915:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828914.823931:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787828914.823935:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828914.824010:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828915.847949:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828915.847964:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787828915.847977:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:0.0:1787828915.848028:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828916.871891:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828916.871895:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787828916.871898:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787828916.871948:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828917.895945:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787828917.895946:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828917.895959:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787828917.896007:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828918.919924:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828918.919924:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787828918.919931:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:0.0:1787828918.919991:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828919.815992:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828919.816004:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828919.816009:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828919.816022:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2355 00000100:00080000:1.0:1787828919.816036:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828919.816044:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787828919.816094:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4f00 x1874672085713280/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828919.816126:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828919.816130:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828919.816132:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828919.816136:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2355 00000100:00080000:1.0:1787828919.816139:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828919.816142:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828919.816155:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828919.816166:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6d00 x1874672085713408/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828919.816177:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828919.816180:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787828919.816182:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828919.816185:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2355 00000100:00080000:1.0:1787828919.816188:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787828919.816192:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085713280:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787828919.816195:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828919.816205:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828919.816216:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4780 x1874672085713536/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828919.816239:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828919.816244:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828919.816246:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828919.816253:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2355 00000100:00080000:1.0:1787828919.816258:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828919.816265:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828919.816272:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:1.0:1787828919.816289:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5a40 x1874672085713664/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828919.816298:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828919.816301:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828919.816303:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828919.816307:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2355 00000100:00080000:1.0:1787828919.816311:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828919.816315:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828919.816320:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:1.0:1787828919.816330:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7840 x1874672085713792/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828919.816338:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828919.816342:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828919.816344:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828919.816347:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2355 00000100:00080000:1.0:1787828919.816351:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828919.816355:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787828919.816359:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085713408:192.168.203.141@tcp:38:kworker.0 00000008:00100000:1.0:1787828919.816362:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828919.816378:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4000 x1874672085713920/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828919.816390:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828919.816393:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787828919.816395:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787828919.816401:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085713536:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787828919.816416:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2355 00000100:00080000:1.0:1787828919.816421:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828919.816427:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828919.816432:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828919.816443:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5e00 x1874672085714048/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828919.816457:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828919.816463:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828919.816467:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787828919.816470:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085713664:192.168.203.141@tcp:8:kworker.0 00000100:00080000:1.0:1787828919.816477:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2355 00000100:00080000:1.0:1787828919.816481:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828919.816486:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828919.816494:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:2.0:1787828919.816502:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085713792:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828919.816530:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7480 x1874672085714176/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828919.816539:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828919.816542:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828919.816544:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828919.816547:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2355 00000100:00080000:1.0:1787828919.816551:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787828919.816557:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085713920:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787828919.816559:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828919.816570:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:1.0:1787828919.816584:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4b40 x1874672085714304/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828919.816622:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085714048:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828919.816646:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085714176:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828919.816677:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085714304:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828919.943956:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787828919.943962:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828919.944013:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828919.944019:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828920.967917:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828920.967926:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828920.967938:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787828920.967969:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828921.991932:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787828921.991938:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787828921.991977:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828921.992042:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1787828922.888125:0:2399:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.141:988 00000400:00000100:3.0:1787828922.888174:0:2399:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.141@tcp at host 192.168.203.141 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1787828922.888193:0:2399:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1787828922.888242:0:2399:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000800:00000100:3.0:1787828922.888258:0:2399:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1787828922.888268:0:2399:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1787828922.888278:0:2399:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1787828922.888286:0:2399:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1787828922.888296:0:2399:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1787828922.888305:0:2399:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1787828922.888314:0:2399:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:2.0:1787828922.888652:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828919/real 1787828922] req@ffff9e35da6f4b40 x1874672085714304/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828944 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828922.888709:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4b40 x1874672085714304/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828944 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828922.888725:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828922.888733:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828922.888739:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085714304:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828922.888750:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828919/real 1787828922] req@ffff9e35da6f4000 x1874672085713920/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828944 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828922.888790:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4000 x1874672085713920/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828944 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828922.888867:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828922.888872:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828922.888877:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085713920:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828922.888887:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828919/real 1787828922] req@ffff9e35da6f5e00 x1874672085714048/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828944 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828922.888906:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5e00 x1874672085714048/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828944 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828922.888916:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828922.888920:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828922.888928:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085714048:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828922.888939:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828919/real 1787828922] req@ffff9e35da6f7480 x1874672085714176/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828944 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828922.888967:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7480 x1874672085714176/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828944 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828922.888980:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828922.888984:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828922.888989:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085714176:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828922.889000:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828919/real 1787828922] req@ffff9e35da6f6d00 x1874672085713408/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828944 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828922.889019:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6d00 x1874672085713408/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828944 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828922.889030:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828922.889035:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828922.889039:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085713408:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828922.889049:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828919/real 1787828922] req@ffff9e35da6f4780 x1874672085713536/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828944 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828922.889065:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4780 x1874672085713536/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828944 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828922.889075:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828922.889080:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828922.889085:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085713536:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828922.889094:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828919/real 1787828922] req@ffff9e35da6f5a40 x1874672085713664/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828944 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828922.889108:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5a40 x1874672085713664/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828944 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828922.889119:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828922.889123:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828922.889127:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085713664:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828922.889136:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828919/real 1787828922] req@ffff9e35da6f7840 x1874672085713792/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828944 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828922.889160:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7840 x1874672085713792/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828944 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828922.889173:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828922.889177:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828922.889181:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085713792:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828922.889189:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828919/real 1787828922] req@ffff9e35da6f4f00 x1874672085713280/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828944 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828922.889204:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4f00 x1874672085713280/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828944 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828922.889213:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828922.889217:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828922.889223:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085713280:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1787828923.015928:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787828923.015978:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828923.016004:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828923.016028:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828924.039904:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787828924.039911:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828924.039939:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787828924.039960:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828924.935955:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828924.935968:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828924.935975:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828924.935989:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2360 00000100:00080000:0.0:1787828924.936003:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828924.936014:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787828924.936072:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7192c0 x1874672085714432/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828924.936102:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828924.936107:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828924.936110:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828924.936116:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2360 00000100:00080000:0.0:1787828924.936122:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828924.936127:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828924.936139:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828924.936144:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085714432:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1787828924.936170:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718000 x1874672085714560/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828924.936184:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828924.936188:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787828924.936191:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828924.936195:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2360 00000100:00080000:0.0:1787828924.936199:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828924.936204:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828924.936210:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828924.936224:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a580 x1874672085714688/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828924.936234:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828924.936238:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787828924.936241:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1787828924.936243:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787828924.936249:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2360 00000100:00080000:0.0:1787828924.936254:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828924.936259:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828924.936265:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:0.0:1787828924.936280:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da719a40 x1874672085714816/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828924.936292:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787828924.936292:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828924/real 1787828924] req@ffff9e35da7192c0 x1874672085714432/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828949 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828924.936300:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787828924.936304:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828924.936314:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2360 00000100:00080000:0.0:1787828924.936319:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828924.936324:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787828924.936329:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7192c0 x1874672085714432/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828949 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1787828924.936332:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00080000:1.0:1787828924.936349:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1787828924.936351:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718780 x1874672085714944/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828924.936357:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828924.936367:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085714432:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1787828924.936373:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828924.936378:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828924.936381:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828924.936387:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2360 00000100:00080000:0.0:1787828924.936400:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1787828924.936401:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085714560:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787828924.936410:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828924.936417:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828924.936432:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a1c0 x1874672085715072/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787828924.936439:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787828924.936462:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828924.936467:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787828924.936470:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787828924.936476:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828924/real 1787828924] req@ffff9e35da718000 x1874672085714560/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828949 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828924.936479:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2360 00000100:00080000:0.0:1787828924.936487:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828924.936496:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787828924.936502:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718000 x1874672085714560/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828949 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1787828924.936503:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00080000:1.0:1787828924.936519:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1787828924.936522:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7183c0 x1874672085715200/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828924.936526:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828924.936536:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085714560:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787828924.936543:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787828924.936548:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085714688:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787828924.936550:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787828924.936556:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828924.936563:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2360 00000100:00080000:0.0:1787828924.936568:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828924.936573:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828924.936579:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000800:00000100:1.0:1787828924.936581:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828924.936607:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828924/real 1787828924] req@ffff9e35da71a580 x1874672085714688/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828949 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828924.936615:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a940 x1874672085715328/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828924.936624:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672085714688/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828949 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828924.936633:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828924.936640:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828924.936640:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828924.936646:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828924.936648:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828924.936665:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085714688:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787828924.936668:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2360 00000100:00080000:0.0:1787828924.936676:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828924.936691:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787828924.936691:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085714816:192.168.203.141@tcp:8:kworker.0 00000008:00100000:0.0:1787828924.936701:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:0.0:1787828924.936719:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718f00 x1874672085715456/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787828924.936727:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828924.936760:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828924/real 1787828924] req@ffff9e35da719a40 x1874672085714816/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828949 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828924.936776:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672085714816/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828949 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828924.936785:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828924.936789:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828924.936792:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085714816:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828924.936831:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085714944:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828924.936853:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828924.936865:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828924/real 1787828924] req@ffff9e35da718780 x1874672085714944/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828949 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828924.936877:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718780 x1874672085714944/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828949 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828924.936887:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828924.936890:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828924.936893:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085714944:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828924.936912:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085715072:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787828924.936933:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828924.936944:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828924/real 1787828924] req@ffff9e35da71a1c0 x1874672085715072/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828949 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828924.936956:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a1c0 x1874672085715072/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828949 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828924.936966:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828924.936969:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828924.936973:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085715072:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828924.936980:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085715200:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787828924.937001:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828924.937013:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828924/real 1787828924] req@ffff9e35da7183c0 x1874672085715200/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828949 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828924.937024:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672085715200/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828949 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828924.937033:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828924.937037:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828924.937041:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085715200:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828924.937047:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085715328:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828924.937066:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828924.937078:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828924/real 1787828924] req@ffff9e35da71a940 x1874672085715328/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828949 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828924.937090:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672085715328/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828949 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828924.937099:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828924.937102:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828924.937105:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085715328:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828924.937111:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085715456:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828924.937133:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828924.937146:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828924/real 1787828924] req@ffff9e35da718f00 x1874672085715456/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828949 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828924.937159:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672085715456/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828949 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828924.937167:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828924.937171:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828924.937174:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085715456:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828925.063899:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787828925.063906:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828925.063935:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828925.063953:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787828926.087927:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828926.087928:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828926.087958:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828926.087981:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828927.111925:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787828927.111973:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828927.111976:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787828927.112073:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828928.135938:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828928.135938:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787828928.136026:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828928.136084:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787828929.159952:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828929.159954:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828929.159958:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787828929.159987:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787828930.055926:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828930.055936:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828930.055941:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828930.055951:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2365 00000100:00080000:1.0:1787828930.055963:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828930.055970:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787828930.056013:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f70c0 x1874672085715584/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828930.056046:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828930.056049:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828930.056050:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828930.056053:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2365 00000100:00080000:1.0:1787828930.056055:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828930.056058:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828930.056068:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828930.056077:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6940 x1874672085715712/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828930.056086:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828930.056088:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787828930.056091:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787828930.056093:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085715584:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787828930.056096:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2365 00000100:00080000:1.0:1787828930.056102:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828930.056106:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828930.056109:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828930.056117:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6580 x1874672085715840/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828930.056123:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828930.056125:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828930.056126:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828930.056129:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2365 00000100:00080000:1.0:1787828930.056132:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828930.056135:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828930.056138:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:1.0:1787828930.056149:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f43c0 x1874672085715968/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828930.056159:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828930.056163:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828930.056165:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828930.056171:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2365 00000100:00080000:1.0:1787828930.056175:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828930.056179:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:2.0:1787828930.056184:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:1.0:1787828930.056187:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:1.0:1787828930.056205:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f61c0 x1874672085716096/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828930.056216:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828930.056219:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828930.056221:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828930.056225:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2365 00000100:00080000:1.0:1787828930.056229:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787828930.056232:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828930/real 1787828930] req@ffff9e35da6f70c0 x1874672085715584/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828955 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828930.056236:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828930.056244:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828930.056256:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f70c0 x1874672085715584/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828955 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828930.056258:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4f00 x1874672085716224/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828930.056281:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828930.056286:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787828930.056290:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828930.056291:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828930.056296:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2365 00000100:00080000:2.0:1787828930.056298:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:1.0:1787828930.056305:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787828930.056305:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085715584:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787828930.056315:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828930.056322:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828930.056336:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7840 x1874672085716352/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828930.056338:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085715712:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787828930.056355:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828930.056360:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828930.056362:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1787828930.056368:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787828930.056371:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2365 00000100:00080000:1.0:1787828930.056378:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828930.056383:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787828930.056390:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828930/real 1787828930] req@ffff9e35da6f6940 x1874672085715712/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828955 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1787828930.056392:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:1.0:1787828930.056411:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5a40 x1874672085716480/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828930.056413:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6940 x1874672085715712/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828955 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828930.056443:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828930.056446:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828930.056452:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085715712:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787828930.056454:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787828930.056459:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085715840:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787828930.056461:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828930.056465:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828930.056470:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2365 00000100:00080000:1.0:1787828930.056474:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828930.056478:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828930.056483:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000800:00000100:2.0:1787828930.056486:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828930.056498:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4780 x1874672085716608/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828930.056500:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828930/real 1787828930] req@ffff9e35da6f6580 x1874672085715840/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828955 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828930.056521:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6580 x1874672085715840/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828955 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828930.056528:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828930.056531:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828930.056533:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085715840:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828930.056540:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085715968:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828930.056557:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828930.056569:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828930/real 1787828930] req@ffff9e35da6f43c0 x1874672085715968/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828955 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828930.056583:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f43c0 x1874672085715968/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828955 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828930.056593:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828930.056597:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828930.056600:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085715968:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828930.056606:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085716096:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828930.056622:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828930.056630:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828930/real 1787828930] req@ffff9e35da6f61c0 x1874672085716096/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828955 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828930.056638:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f61c0 x1874672085716096/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828955 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828930.056644:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828930.056646:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828930.056648:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085716096:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828930.056651:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085716224:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828930.056661:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828930.056667:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828930/real 1787828930] req@ffff9e35da6f4f00 x1874672085716224/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828955 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828930.056674:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4f00 x1874672085716224/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828955 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828930.056679:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828930.056681:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828930.056683:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085716224:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828930.056704:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085716352:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828930.056737:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828930.056745:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828930/real 1787828930] req@ffff9e35da6f7840 x1874672085716352/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828955 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828930.056753:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7840 x1874672085716352/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828955 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828930.056759:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828930.056761:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828930.056763:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085716352:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828930.056767:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085716480:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828930.056777:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828930.056782:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828930/real 1787828930] req@ffff9e35da6f5a40 x1874672085716480/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828955 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828930.056789:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5a40 x1874672085716480/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828955 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828930.056795:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828930.056796:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828930.056826:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085716480:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828930.056839:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085716608:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828930.056854:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828930.056862:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828930/real 1787828930] req@ffff9e35da6f4780 x1874672085716608/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828955 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828930.056870:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4780 x1874672085716608/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828955 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828930.056875:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828930.056877:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828930.056880:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085716608:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787828930.183894:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828930.183894:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828930.183922:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787828930.183928:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828931.207915:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787828931.207916:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828931.207916:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828931.207993:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828932.231903:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:3.0:1787828932.231903:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828932.231953:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828932.232007:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828933.255946:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787828933.255969:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828933.255984:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787828933.255989:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828934.279916:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828934.279917:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787828934.279957:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787828934.279977:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828935.175914:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828935.175922:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828935.175926:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828935.175934:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2370 00000100:00080000:0.0:1787828935.175943:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828935.175949:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787828935.175994:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718b40 x1874672085716736/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828935.176017:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828935.176021:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828935.176022:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828935.176025:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2370 00000100:00080000:0.0:1787828935.176027:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828935.176029:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828935.176037:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828935.176048:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718f00 x1874672085716864/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828935.176054:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828935.176067:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787828935.176068:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828935.176071:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2370 00000100:00080000:0.0:1787828935.176072:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828935.176075:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828935.176078:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828935.176084:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a940 x1874672085716992/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828935.176090:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828935.176094:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787828935.176095:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828935.176098:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2370 00000100:00080000:0.0:1787828935.176100:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828935.176102:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828935.176106:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:0.0:1787828935.176111:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7183c0 x1874672085717120/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828935.176119:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828935.176121:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787828935.176122:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828935.176136:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2370 00000100:00080000:0.0:1787828935.176139:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828935.176143:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828935.176149:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:2.0:1787828935.176155:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085716736:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1787828935.176167:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a1c0 x1874672085717248/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828935.176181:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828935.176185:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828935.176188:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828935.176192:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2370 00000100:00080000:0.0:1787828935.176195:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828935.176200:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828935.176205:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828935.176216:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718780 x1874672085717376/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828935.176223:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828935.176227:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787828935.176228:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828935.176230:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2370 00000100:00080000:0.0:1787828935.176234:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828935.176237:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:2.0:1787828935.176244:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:0.0:1787828935.176247:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828935.176261:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da719a40 x1874672085717504/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828935.176269:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828935.176272:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787828935.176273:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828935.176276:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2370 00000100:00080000:0.0:1787828935.176278:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828935.176281:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828935.176284:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:0.0:1787828935.176292:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a580 x1874672085717632/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828935.176300:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787828935.176304:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828935/real 1787828935] req@ffff9e35da718b40 x1874672085716736/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828960 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828935.176305:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787828935.176311:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828935.176326:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2370 00000100:00080000:0.0:1787828935.176335:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828935.176340:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828935.176344:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:2.0:1787828935.176346:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718b40 x1874672085716736/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828960 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828935.176358:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718000 x1874672085717760/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828935.176360:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828935.176376:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828935.176388:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085716736:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1787828935.176397:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085716864:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828935.176418:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828935.176426:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828935/real 1787828935] req@ffff9e35da718f00 x1874672085716864/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828960 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828935.176434:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672085716864/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828960 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828935.176441:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828935.176443:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828935.176446:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085716864:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828935.176451:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085716992:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828935.176462:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828935.176468:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828935/real 1787828935] req@ffff9e35da71a940 x1874672085716992/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828960 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828935.176475:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672085716992/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828960 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828935.176481:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828935.176483:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828935.176485:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085716992:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828935.176512:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085717120:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828935.176523:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828935.176529:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828935/real 1787828935] req@ffff9e35da7183c0 x1874672085717120/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828960 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828935.176537:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672085717120/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828960 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828935.176543:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828935.176544:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828935.176546:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085717120:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828935.176550:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085717248:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828935.176560:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828935.176566:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828935/real 1787828935] req@ffff9e35da71a1c0 x1874672085717248/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828960 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828935.176574:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a1c0 x1874672085717248/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828960 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828935.176579:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828935.176581:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828935.176583:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085717248:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828935.176587:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085717376:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828935.176597:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828935.176603:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828935/real 1787828935] req@ffff9e35da718780 x1874672085717376/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828960 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828935.176611:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718780 x1874672085717376/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828960 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828935.176616:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828935.176617:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828935.176619:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085717376:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828935.176623:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085717504:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828935.176633:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828935.176639:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828935/real 1787828935] req@ffff9e35da719a40 x1874672085717504/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828960 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828935.176646:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672085717504/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828960 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828935.176652:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828935.176654:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828935.176656:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085717504:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828935.176660:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085717632:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828935.176687:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828935.176713:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828935/real 1787828935] req@ffff9e35da71a580 x1874672085717632/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828960 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828935.176720:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672085717632/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828960 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828935.176727:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828935.176729:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828935.176731:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085717632:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828935.176736:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085717760:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828935.176750:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828935.176774:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828935/real 1787828935] req@ffff9e35da718000 x1874672085717760/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828960 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828935.176783:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718000 x1874672085717760/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828960 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828935.176790:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828935.176793:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828935.176796:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085717760:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787828935.303950:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828935.303954:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828935.303954:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:0.0:1787828935.304021:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828936.327905:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787828936.327931:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:1.0:1787828936.327945:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828936.327955:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787828937.351926:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828937.351945:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787828937.351963:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787828937.352047:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828938.375873:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828938.375874:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:3.0:1787828938.375883:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787828938.375970:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828939.399969:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787828939.400007:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:1.0:1787828939.400048:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828939.400054:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787828940.295878:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828940.295886:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828940.295890:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828940.295899:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2375 00000100:00080000:0.0:1787828940.295909:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828940.295914:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787828940.295948:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7192c0 x1874672085717888/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828940.295973:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828940.295976:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828940.295978:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828940.295981:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2375 00000100:00080000:0.0:1787828940.295983:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828940.295986:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828940.295995:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828940.296007:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718000 x1874672085718016/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828940.296017:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828940.296021:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:2.0:1787828940.296023:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085717888:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1787828940.296024:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828940.296039:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2375 00000100:00080000:0.0:1787828940.296042:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828940.296046:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828940.296050:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828940.296061:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a580 x1874672085718144/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828940.296069:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828940.296071:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787828940.296073:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828940.296076:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2375 00000100:00080000:0.0:1787828940.296081:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828940.296084:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828940.296087:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000800:00000100:2.0:1787828940.296092:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828940.296100:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da719a40 x1874672085718272/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828940.296110:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828940.296112:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787828940.296114:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828940.296116:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2375 00000100:00080000:0.0:1787828940.296119:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828940.296122:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828940.296125:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:2.0:1787828940.296131:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828940/real 1787828940] req@ffff9e35da7192c0 x1874672085717888/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828965 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828940.296136:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718780 x1874672085718400/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828940.296175:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828940.296178:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828940.296180:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787828940.296182:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7192c0 x1874672085717888/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828965 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828940.296186:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2375 00000100:00080000:0.0:1787828940.296192:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828940.296199:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:2.0:1787828940.296200:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:0.0:1787828940.296202:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1787828940.296206:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828940.296211:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085717888:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1787828940.296215:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a1c0 x1874672085718528/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828940.296222:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828940.296224:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787828940.296226:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828940.296228:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2375 00000100:00080000:0.0:1787828940.296230:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828940.296232:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828940.296235:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828940.296237:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085718016:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787828940.296245:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7183c0 x1874672085718656/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828940.296254:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828940.296256:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787828940.296258:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1787828940.296261:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787828940.296262:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2375 00000100:00080000:0.0:1787828940.296268:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828940.296271:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828940.296275:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:2.0:1787828940.296278:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828940/real 1787828940] req@ffff9e35da718000 x1874672085718016/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828965 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828940.296287:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a940 x1874672085718784/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828940.296292:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718000 x1874672085718016/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828965 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828940.296301:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828940.296306:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787828940.296308:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828940.296308:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828940.296317:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2375 00000100:00080000:2.0:1787828940.296318:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:0.0:1787828940.296323:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787828940.296324:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085718016:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787828940.296328:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828940.296332:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:0.0:1787828940.296339:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718f00 x1874672085718912/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828940.296344:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085718144:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828940.296368:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828940.296378:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828940/real 1787828940] req@ffff9e35da71a580 x1874672085718144/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828965 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828940.296388:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672085718144/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828965 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828940.296395:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828940.296398:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828940.296400:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085718144:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828940.296405:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085718272:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828940.296417:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828940.296423:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828940/real 1787828940] req@ffff9e35da719a40 x1874672085718272/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828965 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828940.296430:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672085718272/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828965 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828940.296435:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828940.296437:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828940.296439:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085718272:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828940.296442:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085718400:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828940.296452:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828940.296458:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828940/real 1787828940] req@ffff9e35da718780 x1874672085718400/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828965 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828940.296466:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718780 x1874672085718400/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828965 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828940.296472:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828940.296475:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828940.296480:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085718400:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828940.296503:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085718528:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828940.296525:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828940.296542:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828940/real 1787828940] req@ffff9e35da71a1c0 x1874672085718528/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828965 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828940.296558:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a1c0 x1874672085718528/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828965 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828940.296583:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828940.296587:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828940.296592:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085718528:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828940.296616:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085718656:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828940.296653:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828940.296664:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828940/real 1787828940] req@ffff9e35da7183c0 x1874672085718656/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828965 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828940.296697:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672085718656/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828965 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828940.296704:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828940.296706:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828940.296708:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085718656:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828940.296712:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085718784:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828940.296728:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828940.296735:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828940/real 1787828940] req@ffff9e35da71a940 x1874672085718784/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828965 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828940.296744:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672085718784/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828965 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828940.296750:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828940.296752:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828940.296754:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085718784:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828940.296758:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085718912:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828940.296768:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828940.296775:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828940/real 1787828940] req@ffff9e35da718f00 x1874672085718912/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828965 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828940.296782:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672085718912/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828965 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828940.296793:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828940.296796:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828940.296798:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085718912:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828940.423907:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787828940.423920:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:3.0:1787828940.423952:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787828940.424070:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828941.447921:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:3.0:1787828941.447972:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828941.448004:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:0.0:1787828941.448160:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828942.471961:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828942.471969:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787828942.472018:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787828942.472081:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828943.495914:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828943.495922:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828943.495960:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828943.496015:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828944.519986:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828944.519999:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787828944.520032:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787828944.520088:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828945.415936:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828945.415947:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828945.415952:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828945.415964:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2380 00000100:00080000:0.0:1787828945.415976:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828945.415983:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787828945.416028:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718b40 x1874672085719040/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828945.416060:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828945.416065:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828945.416067:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828945.416072:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2380 00000100:00080000:0.0:1787828945.416077:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828945.416082:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828945.416096:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828945.416111:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718f00 x1874672085719168/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828945.416122:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828945.416126:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787828945.416128:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828945.416132:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2380 00000100:00080000:0.0:1787828945.416134:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828945.416137:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828945.416140:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828945.416160:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a940 x1874672085719296/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828945.416164:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085719040:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1787828945.416188:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828945.416192:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787828945.416194:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828945.416198:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2380 00000100:00080000:0.0:1787828945.416202:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828945.416205:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828945.416212:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:0.0:1787828945.416222:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7183c0 x1874672085719424/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828945.416230:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828945.416233:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787828945.416235:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828945.416239:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2380 00000100:00080000:0.0:1787828945.416241:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828945.416243:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828945.416247:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:0.0:1787828945.416254:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a1c0 x1874672085719552/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828945.416263:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1787828945.416270:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787828945.416272:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828945.416276:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828945.416282:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2380 00000100:00080000:0.0:1787828945.416287:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828945.416292:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828945.416297:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828945.416309:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718780 x1874672085719680/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828945.416317:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828945.416320:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787828945.416322:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828945.416325:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2380 00000100:00080000:0.0:1787828945.416330:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828945.416333:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828945.416337:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828945.416342:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828945/real 1787828945] req@ffff9e35da718b40 x1874672085719040/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828970 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828945.416349:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da719a40 x1874672085719808/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828945.416403:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828945.416407:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787828945.416409:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828945.416418:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2380 00000100:00100000:2.0:1787828945.416420:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718b40 x1874672085719040/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828970 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828945.416426:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828945.416442:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828945.416453:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00080000:2.0:1787828945.416454:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828945.416469:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828945.416476:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085719040:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1787828945.416478:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a580 x1874672085719936/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828945.416496:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787828945.416497:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085719168:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787828945.416512:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787828945.416515:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828945.416520:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2380 00000100:00080000:0.0:1787828945.416527:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828945.416532:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828945.416537:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000800:00000100:2.0:1787828945.416543:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828945.416560:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718000 x1874672085720064/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828945.416570:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828945/real 1787828945] req@ffff9e35da718f00 x1874672085719168/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828970 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828945.416595:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672085719168/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828970 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828945.416628:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828945.416636:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828945.416642:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085719168:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828945.416676:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085719296:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828945.416697:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828945.416708:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828945/real 1787828945] req@ffff9e35da71a940 x1874672085719296/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828970 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828945.416720:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672085719296/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828970 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828945.416728:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828945.416731:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828945.416734:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085719296:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828945.416740:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085719424:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828945.416755:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828945.416764:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828945/real 1787828945] req@ffff9e35da7183c0 x1874672085719424/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828970 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828945.416774:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672085719424/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828970 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828945.416783:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828945.416786:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828945.416790:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085719424:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828945.416795:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085719552:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828945.416861:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828945.416872:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828945/real 1787828945] req@ffff9e35da71a1c0 x1874672085719552/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828970 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828945.416883:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a1c0 x1874672085719552/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828970 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828945.416892:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828945.416895:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828945.416898:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085719552:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828945.416904:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085719680:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828945.416944:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828945.416955:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828945/real 1787828945] req@ffff9e35da718780 x1874672085719680/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828970 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828945.416967:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718780 x1874672085719680/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828970 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828945.416976:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828945.416979:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828945.416983:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085719680:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828945.416990:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085719808:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828945.417007:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828945.417017:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828945/real 1787828945] req@ffff9e35da719a40 x1874672085719808/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828970 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828945.417029:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672085719808/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828970 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828945.417038:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828945.417041:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828945.417044:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085719808:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828945.417050:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085719936:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828945.417067:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828945.417077:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828945/real 1787828945] req@ffff9e35da71a580 x1874672085719936/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828970 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828945.417090:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672085719936/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828970 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828945.417099:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828945.417102:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828945.417105:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085719936:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828945.417111:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085720064:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828945.417127:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828945.417138:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828945/real 1787828945] req@ffff9e35da718000 x1874672085720064/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828970 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828945.417150:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718000 x1874672085720064/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828970 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828945.417158:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828945.417162:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828945.417165:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085720064:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787828945.543903:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828945.543912:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787828945.543976:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828945.543989:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828946.567913:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828946.567967:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787828946.567996:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828946.568055:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828947.591901:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787828947.591903:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828947.591944:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787828947.591952:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828948.615911:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828948.615911:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828948.615917:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787828948.615986:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828949.639886:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828949.639914:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828949.639947:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828949.639968:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787828950.535918:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828950.535930:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787828950.535935:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828950.535947:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2385 00000100:00080000:3.0:1787828950.535961:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828950.535976:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787828950.536032:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a1c0 x1874672085720192/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828950.536062:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828950.536066:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828950.536068:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828950.536071:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2385 00000100:00080000:3.0:1787828950.536075:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828950.536078:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828950.536095:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828950.536108:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a580 x1874672085720320/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828950.536118:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828950.536122:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787828950.536124:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828950.536128:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2385 00000100:00080000:3.0:1787828950.536133:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828950.536136:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828950.536139:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828950.536147:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b0c0 x1874672085720448/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828950.536157:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828950.536161:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787828950.536163:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828950.536168:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2385 00000100:00080000:3.0:1787828950.536175:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787828950.536177:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085720192:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1787828950.536182:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828950.536196:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:3.0:1787828950.536207:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b480 x1874672085720576/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828950.536215:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828950.536220:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787828950.536222:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828950.536226:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2385 00000100:00080000:3.0:1787828950.536228:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828950.536232:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828950.536238:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:3.0:1787828950.536249:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c64983c0 x1874672085720704/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828950.536258:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828950.536263:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828950.536264:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828950.536268:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2385 00000100:00080000:3.0:1787828950.536274:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1787828950.536279:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1787828950.536285:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828950.536292:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828950.536307:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499e00 x1874672085720832/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828950.536315:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828950.536318:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787828950.536320:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828950.536328:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2385 00000100:00080000:3.0:1787828950.536333:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828950.536338:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828950.536343:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828950.536345:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828950/real 1787828950] req@ffff9e35c649a1c0 x1874672085720192/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828975 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828950.536357:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c64992c0 x1874672085720960/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828950.536374:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828950.536383:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:2.0:1787828950.536384:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a1c0 x1874672085720192/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828975 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828950.536393:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828950.536415:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2385 00000100:00080000:2.0:1787828950.536417:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828950.536426:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828950.536427:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828950.536437:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085720192:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1787828950.536443:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787828950.536452:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085720320:192.168.203.141@tcp:38:kworker.0 00000008:00100000:3.0:1787828950.536454:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:3.0:1787828950.536483:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498000 x1874672085721088/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787828950.536496:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1787828950.536507:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828950.536512:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787828950.536515:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787828950.536522:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828950/real 1787828950] req@ffff9e35c649a580 x1874672085720320/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828975 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828950.536528:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2385 00000100:00080000:3.0:1787828950.536548:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787828950.536553:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a580 x1874672085720320/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828975 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828950.536578:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828950.536589:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00080000:2.0:1787828950.536593:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828950.536599:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828950.536607:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085720320:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787828950.536608:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649ad00 x1874672085721216/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828950.536663:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085720448:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828950.536686:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828950.536696:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828950/real 1787828950] req@ffff9e35c649b0c0 x1874672085720448/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828975 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828950.536707:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b0c0 x1874672085720448/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828975 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828950.536715:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828950.536718:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828950.536721:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085720448:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828950.536727:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085720576:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828950.536741:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828950.536749:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828950/real 1787828950] req@ffff9e35c649b480 x1874672085720576/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828975 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828950.536760:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b480 x1874672085720576/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828975 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828950.536774:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828950.536777:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828950.536780:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085720576:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828950.536785:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c64983c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085720704:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828950.536830:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828950.536841:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828950/real 1787828950] req@ffff9e35c64983c0 x1874672085720704/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828975 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828950.536852:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c64983c0 x1874672085720704/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828975 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828950.536860:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828950.536863:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828950.536866:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c64983c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085720704:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828950.536872:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085720832:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828950.536890:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828950.536899:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828950/real 1787828950] req@ffff9e35c6499e00 x1874672085720832/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828975 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828950.536909:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499e00 x1874672085720832/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828975 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828950.536917:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828950.536920:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828950.536923:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085720832:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828950.536929:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c64992c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085720960:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828950.536943:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828950.536951:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828950/real 1787828950] req@ffff9e35c64992c0 x1874672085720960/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828975 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828950.536960:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c64992c0 x1874672085720960/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828975 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828950.536967:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828950.536970:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828950.536972:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c64992c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085720960:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828950.536977:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085721088:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828950.536991:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828950.536998:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828950/real 1787828950] req@ffff9e35c6498000 x1874672085721088/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828975 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828950.537008:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498000 x1874672085721088/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828975 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828950.537015:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828950.537018:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828950.537020:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085721088:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828950.537025:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085721216:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828950.537038:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828950.537046:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828950/real 1787828950] req@ffff9e35c649ad00 x1874672085721216/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828975 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828950.537056:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649ad00 x1874672085721216/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828975 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828950.537063:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828950.537066:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828950.537068:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085721216:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787828950.663916:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828950.663916:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828950.663963:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:1.0:1787828950.663990:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828951.687922:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828951.687922:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828951.687952:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:1.0:1787828951.687977:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828952.711931:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828952.711935:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828952.711975:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828952.712001:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828953.735945:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787828953.735946:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828953.735946:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787828953.736002:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828954.759976:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787828954.759983:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828954.759991:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828954.760006:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828955.655940:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828955.655949:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787828955.655968:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828955.655976:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2391 00000100:00080000:3.0:1787828955.655987:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828955.655993:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787828955.656028:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499a40 x1874672085721344/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828955.656050:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828955.656053:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828955.656055:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828955.656059:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2391 00000100:00080000:3.0:1787828955.656079:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828955.656082:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787828955.656089:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085721344:192.168.203.141@tcp:250:kworker.0 00000008:00100000:3.0:1787828955.656093:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828955.656108:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498780 x1874672085721472/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828955.656117:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828955.656120:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787828955.656121:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828955.656124:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2391 00000100:00080000:3.0:1787828955.656127:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828955.656130:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828955.656133:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828955.656143:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a940 x1874672085721600/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828955.656151:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828955.656154:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787828955.656157:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1787828955.656165:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1787828955.656165:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2391 00000100:00080000:3.0:1787828955.656175:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828955.656181:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828955.656214:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:3.0:1787828955.656230:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498b40 x1874672085721728/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828955.656231:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828955/real 1787828955] req@ffff9e35c6499a40 x1874672085721344/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828980 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828955.656249:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828955.656253:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787828955.656255:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828955.656259:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2391 00000100:00080000:3.0:1787828955.656262:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787828955.656263:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499a40 x1874672085721344/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828980 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828955.656268:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828955.656274:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00080000:2.0:1787828955.656285:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1787828955.656287:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499680 x1874672085721856/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828955.656296:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:3.0:1787828955.656299:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828955.656304:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:2.0:1787828955.656305:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085721344:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1787828955.656306:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828955.656313:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2391 00000100:00080000:3.0:1787828955.656315:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828955.656318:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828955.656320:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828955.656330:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498f00 x1874672085721984/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828955.656340:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828955.656344:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787828955.656346:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787828955.656351:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085721472:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1787828955.656352:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2391 00000100:00080000:3.0:1787828955.656360:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828955.656366:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828955.656371:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828955.656383:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b840 x1874672085722112/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787828955.656388:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1787828955.656399:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828955.656402:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787828955.656403:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787828955.656406:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828955/real 1787828955] req@ffff9e35c6498780 x1874672085721472/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828980 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828955.656409:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2391 00000100:00080000:3.0:1787828955.656433:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828955.656441:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828955.656447:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:2.0:1787828955.656448:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498780 x1874672085721472/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828980 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828955.656461:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1787828955.656462:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649bc00 x1874672085722240/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828955.656467:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828955.656477:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085721472:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1787828955.656483:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828955.656487:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787828955.656488:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828955.656492:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2391 00000100:00080000:3.0:1787828955.656495:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787828955.656499:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085721600:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1787828955.656501:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828955.656510:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:3.0:1787828955.656522:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499a40 x1874672085722368/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787828955.656547:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828955.656563:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828955/real 1787828955] req@ffff9e35c649a940 x1874672085721600/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828980 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828955.656591:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a940 x1874672085721600/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828980 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828955.656598:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828955.656601:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828955.656604:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085721600:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828955.656608:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085721728:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828955.656631:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828955.656645:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828955/real 1787828955] req@ffff9e35c6498b40 x1874672085721728/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828980 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828955.656660:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498b40 x1874672085721728/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828980 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828955.656671:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828955.656675:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828955.656678:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085721728:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828955.656685:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085721856:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828955.656709:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828955.656723:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828955/real 1787828955] req@ffff9e35c6499680 x1874672085721856/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828980 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828955.656738:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499680 x1874672085721856/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828980 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828955.656748:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828955.656750:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828955.656753:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085721856:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828955.656758:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085721984:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828955.656791:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828955.656801:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828955/real 1787828955] req@ffff9e35c6498f00 x1874672085721984/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828980 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828955.656810:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498f00 x1874672085721984/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828980 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828955.656817:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828955.656819:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828955.656822:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085721984:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828955.656893:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085722112:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828955.656916:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828955.657021:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828955/real 1787828955] req@ffff9e35c649b840 x1874672085722112/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828980 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828955.657033:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b840 x1874672085722112/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828980 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828955.657043:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828955.657047:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828955.657052:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085722112:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828955.657075:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085722240:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828955.657101:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828955.657115:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828955/real 1787828955] req@ffff9e35c649bc00 x1874672085722240/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828980 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828955.657128:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649bc00 x1874672085722240/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828980 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828955.657135:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828955.657138:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828955.657141:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085722240:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828955.657145:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085722368:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828955.657163:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828955.657177:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828955/real 1787828955] req@ffff9e35c6499a40 x1874672085722368/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828980 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828955.657191:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499a40 x1874672085722368/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828980 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828955.657202:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828955.657206:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828955.657210:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085722368:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787828955.783880:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787828955.783902:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787828955.783937:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828955.783997:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828956.807879:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787828956.807899:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787828956.807930:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787828956.807938:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828957.831868:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787828957.831877:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828957.831902:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:0.0:1787828957.831919:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828958.855874:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828958.855875:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828958.855947:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828958.855986:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828959.879940:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787828959.879945:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787828959.879968:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787828959.880009:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828960.775944:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828960.775956:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787828960.775961:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828960.775973:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2396 00000100:00080000:3.0:1787828960.775985:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828960.775992:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787828960.776036:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649ad00 x1874672085722496/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828960.776081:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828960.776085:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828960.776086:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828960.776090:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2396 00000100:00080000:3.0:1787828960.776092:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828960.776095:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828960.776107:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828960.776119:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498000 x1874672085722624/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828960.776159:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787828960.776163:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085722496:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1787828960.776165:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787828960.776168:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828960.776188:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2396 00000100:00080000:3.0:1787828960.776196:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828960.776205:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828960.776225:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1787828960.776249:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787828960.776252:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c64992c0 x1874672085722752/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828960.776268:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828960.776272:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787828960.776275:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828960.776281:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2396 00000100:00080000:3.0:1787828960.776287:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787828960.776295:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828960/real 1787828960] req@ffff9e35c649ad00 x1874672085722496/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828985 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828960.776296:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828960.776315:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:2.0:1787828960.776335:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649ad00 x1874672085722496/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828985 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828960.776350:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828960.776354:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828960.776372:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085722496:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1787828960.776373:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499e00 x1874672085722880/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828960.776403:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787828960.776404:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085722624:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1787828960.776429:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787828960.776435:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828960.776444:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2396 00000100:00080000:3.0:1787828960.776452:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1787828960.776455:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1787828960.776462:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787828960.776472:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828960/real 1787828960] req@ffff9e35c6498000 x1874672085722624/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828985 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1787828960.776473:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:2.0:1787828960.776516:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498000 x1874672085722624/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828985 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828960.776534:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828960.776537:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828960.776539:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085722624:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787828960.776547:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c64983c0 x1874672085723008/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828960.776549:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c64992c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085722752:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1787828960.776571:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828960.776575:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828960.776577:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828960.776585:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2396 00000800:00000100:2.0:1787828960.776588:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1787828960.776591:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828960.776598:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787828960.776604:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828960/real 1787828960] req@ffff9e35c64992c0 x1874672085722752/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828985 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1787828960.776607:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828960.776627:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b480 x1874672085723136/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828960.776641:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c64992c0 x1874672085722752/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828985 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828960.776643:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828960.776657:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787828960.776663:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828960.776676:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2396 00000100:00080000:2.0:1787828960.776677:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828960.776685:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828960.776688:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:3.0:1787828960.776694:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787828960.776697:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c64992c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085722752:192.168.203.141@tcp:38:kworker.0 00000008:00100000:3.0:1787828960.776701:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828960.776715:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b0c0 x1874672085723264/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828960.776722:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085722880:192.168.203.141@tcp:8:kworker.0 00000100:00080000:3.0:1787828960.776750:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828960.776753:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787828960.776755:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828960.776759:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2396 00000800:00000100:2.0:1787828960.776764:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1787828960.776784:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787828960.776786:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828960/real 1787828960] req@ffff9e35c6499e00 x1874672085722880/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828985 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828960.776795:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828960.776803:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:3.0:1787828960.776847:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a580 x1874672085723392/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828960.776864:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828960.776874:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787828960.776876:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828960.776883:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2396 00000100:00080000:3.0:1787828960.776887:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787828960.776893:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499e00 x1874672085722880/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828985 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828960.776898:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828960.776913:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00080000:2.0:1787828960.776918:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828960.776922:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787828960.776927:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a1c0 x1874672085723520/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828960.776928:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085722880:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828960.776957:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c64983c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085723008:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828960.776984:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828960.776994:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828960/real 1787828960] req@ffff9e35c64983c0 x1874672085723008/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828985 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828960.777010:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c64983c0 x1874672085723008/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828985 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828960.777016:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828960.777018:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828960.777023:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c64983c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085723008:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828960.777029:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085723136:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828960.777048:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828960.777069:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828960/real 1787828960] req@ffff9e35c649b480 x1874672085723136/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828985 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828960.777081:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b480 x1874672085723136/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828985 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828960.777090:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828960.777092:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828960.777096:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085723136:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828960.777110:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085723264:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828960.777131:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828960.777148:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828960/real 1787828960] req@ffff9e35c649b0c0 x1874672085723264/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828985 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828960.777155:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b0c0 x1874672085723264/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828985 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828960.777170:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828960.777173:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828960.777176:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085723264:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828960.777183:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085723392:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828960.777201:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828960.777223:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828960/real 1787828960] req@ffff9e35c649a580 x1874672085723392/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828985 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828960.777234:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a580 x1874672085723392/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828985 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828960.777242:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828960.777246:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828960.777253:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085723392:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828960.777258:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085723520:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828960.777274:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828960.777280:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828960/real 1787828960] req@ffff9e35c649a1c0 x1874672085723520/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828985 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828960.777287:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a1c0 x1874672085723520/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828985 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828960.777293:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828960.777295:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828960.777298:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085723520:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787828960.903888:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828960.903927:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787828960.903944:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787828960.903971:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828961.927889:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828961.927920:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828961.927922:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828961.927970:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828962.951904:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828962.951942:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828962.951968:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787828962.951973:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828963.975891:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828963.975900:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:3.0:1787828963.975927:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787828963.976019:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828964.999880:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828964.999889:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828964.999946:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828965.000033:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787828965.895978:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828965.895988:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828965.895993:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828965.896003:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2401 00000100:00080000:0.0:1787828965.896015:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828965.896040:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787828965.896092:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7192c0 x1874672085723648/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828965.896136:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828965.896140:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828965.896142:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828965.896146:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2401 00000100:00080000:0.0:1787828965.896150:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828965.896153:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828965.896163:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828965.896178:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718000 x1874672085723776/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828965.896188:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828965.896192:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787828965.896193:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828965.896197:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2401 00000100:00080000:0.0:1787828965.896199:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828965.896202:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828965.896207:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828965.896216:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085723648:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1787828965.896219:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a580 x1874672085723904/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828965.896254:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828965.896257:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787828965.896260:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828965.896265:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2401 00000100:00080000:0.0:1787828965.896268:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828965.896272:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828965.896278:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:0.0:1787828965.896290:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da719a40 x1874672085724032/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828965.896327:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828965.896331:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787828965.896333:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828965.896337:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2401 00000100:00080000:0.0:1787828965.896341:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1787828965.896349:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787828965.896350:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828965.896361:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:0.0:1787828965.896394:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718780 x1874672085724160/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828965.896406:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828965.896410:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828965.896413:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828965.896419:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2401 00000100:00080000:0.0:1787828965.896425:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787828965.896433:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828965/real 1787828965] req@ffff9e35da7192c0 x1874672085723648/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828990 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828965.896435:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828965.896446:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828965.896524:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7192c0 x1874672085723648/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828990 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828965.896529:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a1c0 x1874672085724288/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828965.896548:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828965.896557:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828965.896557:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828965.896567:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085723648:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1787828965.896569:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787828965.896576:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787828965.896579:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085723776:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787828965.896589:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2401 00000100:00080000:0.0:1787828965.896598:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828965.896605:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:2.0:1787828965.896614:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:0.0:1787828965.896624:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828965.896630:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828965/real 1787828965] req@ffff9e35da718000 x1874672085723776/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828990 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828965.896645:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718000 x1874672085723776/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828990 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828965.896652:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7183c0 x1874672085724416/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828965.896658:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828965.896670:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828965.896679:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085723776:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787828965.896682:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828965.896689:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787828965.896692:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828965.896701:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2401 00000100:00080000:0.0:1787828965.896708:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787828965.896712:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085723904:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787828965.896720:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828965.896730:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000800:00000100:2.0:1787828965.896739:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828965.896753:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a940 x1874672085724544/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828965.896756:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828965/real 1787828965] req@ffff9e35da71a580 x1874672085723904/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828990 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828965.896792:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787828965.896801:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672085723904/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828990 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828965.896810:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828965.896856:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787828965.896858:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:0.0:1787828965.896862:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787828965.896868:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085723904:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787828965.896878:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2401 00000100:00100000:2.0:1787828965.896887:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085724032:192.168.203.141@tcp:8:kworker.0 00000100:00080000:0.0:1787828965.896888:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828965.896902:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787828965.896942:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000800:00000100:2.0:1787828965.896944:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828965.896965:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828965/real 1787828965] req@ffff9e35da719a40 x1874672085724032/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828990 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828965.896971:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718f00 x1874672085724672/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828965.896986:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672085724032/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828990 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828965.896998:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828965.897002:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828965.897006:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085724032:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828965.897013:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085724160:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828965.897034:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828965.897043:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828965/real 1787828965] req@ffff9e35da718780 x1874672085724160/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828990 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828965.897053:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718780 x1874672085724160/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828990 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828965.897061:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828965.897064:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828965.897066:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085724160:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828965.897071:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085724288:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828965.897085:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828965.897093:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828965/real 1787828965] req@ffff9e35da71a1c0 x1874672085724288/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828990 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828965.897110:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a1c0 x1874672085724288/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828990 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828965.897121:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828965.897126:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828965.897131:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085724288:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828965.897139:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085724416:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828965.897169:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828965.897187:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828965/real 1787828965] req@ffff9e35da7183c0 x1874672085724416/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828990 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828965.897205:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672085724416/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828990 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828965.897218:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828965.897222:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828965.897225:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085724416:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828965.897261:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085724544:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828965.897282:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828965.897292:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828965/real 1787828965] req@ffff9e35da71a940 x1874672085724544/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828990 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828965.897303:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672085724544/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828990 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828965.897324:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828965.897328:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828965.897333:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085724544:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828965.897341:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085724672:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828965.897367:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828965.897384:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828965/real 1787828965] req@ffff9e35da718f00 x1874672085724672/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828990 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828965.897402:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672085724672/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828990 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828965.897414:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828965.897418:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828965.897423:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085724672:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828966.023923:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828966.023976:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787828966.024089:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787828966.024215:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828967.047895:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787828967.047895:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828967.047899:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787828967.047994:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828968.071901:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:3.0:1787828968.071901:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787828968.071926:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828968.071973:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828969.095932:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828969.095932:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:3.0:1787828969.095932:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787828969.096075:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828970.119916:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828970.119916:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828970.119974:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828970.119980:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00080000:2.0:1787828971.015943:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828971.015952:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828971.015956:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828971.015966:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2406 00000100:00080000:2.0:1787828971.015975:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828971.015982:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787828971.016017:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498d680 x1874672085724800/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828971.016047:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828971.016050:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828971.016052:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828971.016055:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2406 00000100:00080000:2.0:1787828971.016058:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828971.016062:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828971.016071:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828971.016082:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c780 x1874672085724928/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828971.016090:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828971.016094:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787828971.016097:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828971.016100:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2406 00000100:00080000:2.0:1787828971.016102:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828971.016107:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828971.016111:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828971.016119:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e940 x1874672085725056/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828971.016130:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828971.016134:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787828971.016137:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787828971.016144:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085724800:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1787828971.016147:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2406 00000100:00080000:2.0:1787828971.016158:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828971.016164:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828971.016172:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:2.0:1787828971.016192:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498fc00 x1874672085725184/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828971.016207:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828971.016211:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787828971.016213:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828971.016217:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2406 00000100:00080000:2.0:1787828971.016224:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828971.016228:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828971.016233:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000800:00000100:1.0:1787828971.016237:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828971.016253:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498de00 x1874672085725312/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828971.016265:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828971.016268:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828971.016269:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828971.016274:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2406 00000100:00080000:2.0:1787828971.016277:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828971.016283:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828971.016290:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828971.016293:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828971/real 1787828971] req@ffff9e35c498d680 x1874672085724800/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828996 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828971.016315:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498ed00 x1874672085725440/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828971.016318:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498d680 x1874672085724800/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828996 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828971.016400:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828971.016401:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828971.016413:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787828971.016414:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:2.0:1787828971.016437:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787828971.016438:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085724800:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1787828971.016450:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2406 00000100:00080000:2.0:1787828971.016456:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1787828971.016458:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085724928:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1787828971.016466:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828971.016474:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1787828971.016486:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828971.016491:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c3c0 x1874672085725568/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828971.016501:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828971/real 1787828971] req@ffff9e35c498c780 x1874672085724928/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828996 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828971.016514:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828971.016536:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:1.0:1787828971.016541:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c780 x1874672085724928/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828996 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828971.016541:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828971.016559:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2406 00000100:00080000:1.0:1787828971.016574:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828971.016576:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828971.016580:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828971.016586:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085724928:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1787828971.016594:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828971.016606:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:1.0:1787828971.016628:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085725056:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828971.016630:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498d680 x1874672085725696/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787828971.016656:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1787828971.016674:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828971.016683:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:1.0:1787828971.016684:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828971/real 1787828971] req@ffff9e35c498e940 x1874672085725056/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828996 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828971.016691:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828971.016710:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2406 00000100:00100000:1.0:1787828971.016718:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e940 x1874672085725056/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828996 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828971.016720:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828971.016735:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:1.0:1787828971.016748:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000008:00100000:2.0:1787828971.016749:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00080000:1.0:1787828971.016757:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828971.016761:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085725056:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828971.016771:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085725184:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828971.016776:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cf00 x1874672085725824/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787828971.016789:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828971.016833:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828971/real 1787828971] req@ffff9e35c498fc00 x1874672085725184/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828996 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828971.016843:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498fc00 x1874672085725184/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828996 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828971.016849:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828971.016853:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828971.016855:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085725184:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828971.016862:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085725312:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828971.016878:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828971.016886:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828971/real 1787828971] req@ffff9e35c498de00 x1874672085725312/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828996 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828971.016893:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498de00 x1874672085725312/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828996 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828971.016904:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828971.016906:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828971.016908:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085725312:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828971.016912:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085725440:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787828971.016923:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828971.016929:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828971/real 1787828971] req@ffff9e35c498ed00 x1874672085725440/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828996 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828971.016936:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498ed00 x1874672085725440/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828996 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828971.016942:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828971.016943:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828971.016945:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085725440:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828971.016949:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085725568:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787828971.016959:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828971.016964:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828971/real 1787828971] req@ffff9e35c498c3c0 x1874672085725568/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828996 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828971.016971:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c3c0 x1874672085725568/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828996 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828971.016976:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828971.016978:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828971.016981:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085725568:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828971.016995:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085725696:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828971.017006:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828971.017012:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828971/real 1787828971] req@ffff9e35c498d680 x1874672085725696/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828996 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828971.017020:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498d680 x1874672085725696/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828996 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828971.017026:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828971.017028:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828971.017029:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085725696:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828971.017033:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085725824:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828971.017043:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828971.017048:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828971/real 1787828971] req@ffff9e35c498cf00 x1874672085725824/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828996 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828971.017056:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498cf00 x1874672085725824/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828996 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828971.017061:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828971.017063:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828971.017065:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085725824:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828971.143924:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787828971.143933:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828971.143981:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828971.144052:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828972.167937:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828972.167938:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828972.167962:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787828972.168043:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828973.191887:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828973.191920:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787828973.191928:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828973.192005:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828974.215937:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828974.215972:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787828974.215991:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828974.216087:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828975.239854:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828975.239868:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787828975.239947:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828975.239950:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00080000:3.0:1787828976.135918:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828976.135929:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787828976.135933:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828976.135945:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2411 00000100:00080000:3.0:1787828976.135957:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828976.135964:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787828976.136009:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a1c0 x1874672085725952/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828976.136038:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828976.136042:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828976.136043:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828976.136047:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2411 00000100:00080000:3.0:1787828976.136049:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828976.136052:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828976.136062:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828976.136073:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a580 x1874672085726080/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828976.136083:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828976.136087:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787828976.136089:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828976.136094:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2411 00000100:00080000:3.0:1787828976.136100:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828976.136103:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828976.136117:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828976.136125:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085725952:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1787828976.136126:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b0c0 x1874672085726208/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828976.136136:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828976.136138:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787828976.136140:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828976.136145:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2411 00000100:00080000:3.0:1787828976.136152:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828976.136162:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828976.136165:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:3.0:1787828976.136172:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b480 x1874672085726336/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828976.136178:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828976.136181:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787828976.136182:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828976.136186:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2411 00000100:00080000:3.0:1787828976.136199:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828976.136202:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828976.136207:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:3.0:1787828976.136216:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c64983c0 x1874672085726464/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828976.136223:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828976.136226:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828976.136227:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828976.136231:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2411 00000100:00080000:3.0:1787828976.136234:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828976.136238:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828976.136240:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828976.136248:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499e00 x1874672085726592/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828976.136254:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828976.136257:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787828976.136259:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828976.136261:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2411 00000100:00080000:3.0:1787828976.136263:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828976.136271:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:1.0:1787828976.136272:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:3.0:1787828976.136275:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828976.136285:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c64992c0 x1874672085726720/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828976.136293:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828976.136296:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787828976.136298:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828976.136302:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2411 00000100:00080000:3.0:1787828976.136305:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828976.136308:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828976.136311:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:3.0:1787828976.136320:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498000 x1874672085726848/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828976.136325:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828976.136328:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787828976.136329:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828976.136333:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2411 00000100:00080000:3.0:1787828976.136335:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828976.136339:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828976.136343:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:1.0:1787828976.136351:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828976/real 1787828976] req@ffff9e35c649a1c0 x1874672085725952/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829001 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828976.136356:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649ad00 x1874672085726976/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828976.136431:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a1c0 x1874672085725952/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829001 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828976.136450:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828976.136457:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828976.136464:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085725952:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1787828976.136525:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085726080:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787828976.136583:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828976.136604:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828976/real 1787828976] req@ffff9e35c649a580 x1874672085726080/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829001 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828976.136650:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a580 x1874672085726080/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829001 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828976.136667:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828976.136674:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828976.136679:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085726080:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828976.136688:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085726208:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787828976.136718:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828976.136732:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828976/real 1787828976] req@ffff9e35c649b0c0 x1874672085726208/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829001 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828976.136747:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b0c0 x1874672085726208/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829001 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828976.136756:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828976.136760:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828976.136764:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085726208:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828976.136771:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085726336:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828976.136801:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828976.136820:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828976/real 1787828976] req@ffff9e35c649b480 x1874672085726336/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829001 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828976.136856:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b480 x1874672085726336/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829001 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828976.136904:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828976.136910:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828976.136915:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085726336:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828976.136926:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c64983c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085726464:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828976.136956:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828976.136972:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828976/real 1787828976] req@ffff9e35c64983c0 x1874672085726464/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829001 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828976.136990:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c64983c0 x1874672085726464/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829001 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828976.136999:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828976.137002:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828976.137005:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c64983c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085726464:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828976.137010:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085726592:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787828976.137029:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828976.137038:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828976/real 1787828976] req@ffff9e35c6499e00 x1874672085726592/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829001 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828976.137048:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499e00 x1874672085726592/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829001 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828976.137055:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828976.137057:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828976.137060:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085726592:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828976.137065:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c64992c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085726720:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787828976.137078:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828976.137086:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828976/real 1787828976] req@ffff9e35c64992c0 x1874672085726720/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829001 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828976.137095:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c64992c0 x1874672085726720/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829001 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828976.137127:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828976.137131:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828976.137135:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c64992c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085726720:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828976.137142:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085726848:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828976.137161:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828976.137172:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828976/real 1787828976] req@ffff9e35c6498000 x1874672085726848/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829001 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828976.137184:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498000 x1874672085726848/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829001 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828976.137194:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828976.137197:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828976.137200:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085726848:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828976.137208:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085726976:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828976.137224:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828976.137235:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828976/real 1787828976] req@ffff9e35c649ad00 x1874672085726976/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829001 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828976.137248:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649ad00 x1874672085726976/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829001 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828976.137257:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828976.137260:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828976.137264:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085726976:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828976.263907:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787828976.263918:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828976.263965:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828976.264058:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828977.287917:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828977.287918:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828977.287977:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828977.287983:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828978.311894:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787828978.311936:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828978.311938:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:1.0:1787828978.312037:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828979.335873:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828979.335929:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828979.335948:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828979.335950:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787828980.359906:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787828980.359912:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828980.359939:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828980.359953:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828981.255875:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828981.255883:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787828981.255887:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828981.255895:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2416 00000100:00080000:3.0:1787828981.255904:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828981.255911:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787828981.255946:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499a40 x1874672085727104/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828981.255975:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828981.255979:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828981.255981:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828981.255986:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2416 00000100:00080000:3.0:1787828981.255990:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828981.255994:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828981.256004:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828981.256010:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085727104:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1787828981.256018:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649bc00 x1874672085727232/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828981.256028:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828981.256031:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787828981.256032:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828981.256036:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2416 00000100:00080000:3.0:1787828981.256038:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828981.256041:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828981.256045:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828981.256053:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b840 x1874672085727360/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828981.256060:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828981.256062:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787828981.256065:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828981.256069:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2416 00000100:00080000:3.0:1787828981.256072:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1787828981.256075:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1787828981.256077:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828981.256082:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:3.0:1787828981.256091:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498f00 x1874672085727488/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828981.256097:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828981.256100:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787828981.256101:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828981.256104:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2416 00000100:00080000:3.0:1787828981.256107:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828981.256110:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828981.256113:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:1.0:1787828981.256114:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828981/real 1787828981] req@ffff9e35c6499a40 x1874672085727104/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829006 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828981.256126:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499680 x1874672085727616/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828981.256131:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499a40 x1874672085727104/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829006 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828981.256139:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828981.256146:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828981.256149:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828981.256154:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2416 00000100:00080000:1.0:1787828981.256155:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828981.256157:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828981.256163:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:1.0:1787828981.256166:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000008:00100000:3.0:1787828981.256168:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828981.256174:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085727104:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1787828981.256179:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498b40 x1874672085727744/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828981.256187:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828981.256189:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787828981.256190:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828981.256193:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2416 00000100:00080000:3.0:1787828981.256195:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828981.256198:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828981.256201:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828981.256208:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085727232:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787828981.256210:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a940 x1874672085727872/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828981.256222:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828981.256224:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787828981.256225:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828981.256228:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2416 00000100:00080000:3.0:1787828981.256231:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828981.256233:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828981.256236:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:3.0:1787828981.256247:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498780 x1874672085728000/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787828981.256248:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1787828981.256257:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828981.256260:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787828981.256261:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828981.256265:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2416 00000100:00080000:3.0:1787828981.256267:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1787828981.256270:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828981/real 1787828981] req@ffff9e35c649bc00 x1874672085727232/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829006 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828981.256273:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828981.256279:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:3.0:1787828981.256290:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499a40 x1874672085728128/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828981.256297:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649bc00 x1874672085727232/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829006 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828981.256311:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828981.256316:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828981.256321:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085727232:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828981.256329:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085727360:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787828981.256364:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828981.256378:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828981/real 1787828981] req@ffff9e35c649b840 x1874672085727360/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829006 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828981.256386:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b840 x1874672085727360/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829006 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828981.256393:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828981.256395:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828981.256397:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085727360:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828981.256401:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085727488:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828981.256416:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828981.256423:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828981/real 1787828981] req@ffff9e35c6498f00 x1874672085727488/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829006 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828981.256430:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498f00 x1874672085727488/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829006 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828981.256436:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828981.256437:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828981.256440:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085727488:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828981.256443:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085727616:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828981.256453:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828981.256458:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828981/real 1787828981] req@ffff9e35c6499680 x1874672085727616/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829006 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828981.256465:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499680 x1874672085727616/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829006 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828981.256470:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828981.256472:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828981.256474:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085727616:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828981.256477:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085727744:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787828981.256487:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828981.256493:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828981/real 1787828981] req@ffff9e35c6498b40 x1874672085727744/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829006 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828981.256501:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498b40 x1874672085727744/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829006 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828981.256506:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828981.256508:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828981.256510:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085727744:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828981.256524:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085727872:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787828981.256535:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828981.256541:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828981/real 1787828981] req@ffff9e35c649a940 x1874672085727872/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829006 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828981.256549:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a940 x1874672085727872/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829006 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828981.256554:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828981.256555:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828981.256557:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085727872:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828981.256561:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085728000:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828981.256570:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828981.256576:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828981/real 1787828981] req@ffff9e35c6498780 x1874672085728000/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829006 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828981.256583:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498780 x1874672085728000/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829006 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828981.256588:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828981.256590:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828981.256592:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085728000:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828981.256595:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085728128:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787828981.256604:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828981.256609:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828981/real 1787828981] req@ffff9e35c6499a40 x1874672085728128/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829006 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828981.256616:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499a40 x1874672085728128/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829006 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828981.256622:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828981.256624:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787828981.256626:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085728128:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828981.383905:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828981.383920:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828981.383934:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787828981.383978:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828982.407912:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828982.407912:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787828982.407923:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828982.407958:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828983.431909:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828983.431910:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787828983.431927:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828983.431944:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828984.455915:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787828984.455934:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787828984.455967:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:1.0:1787828984.456034:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828985.479938:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787828985.479956:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828985.480000:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828985.480002:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787828986.375923:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828986.375933:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787828986.375937:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828986.375958:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2421 00000100:00080000:3.0:1787828986.375969:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828986.375977:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787828986.376017:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649ad00 x1874672085728256/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828986.376047:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828986.376051:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828986.376053:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828986.376058:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2421 00000100:00080000:3.0:1787828986.376062:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828986.376066:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828986.376077:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828986.376094:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498000 x1874672085728384/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828986.376105:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828986.376120:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787828986.376122:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828986.376132:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2421 00000100:00100000:1.0:1787828986.376136:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085728256:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1787828986.376138:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828986.376151:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828986.376158:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828986.376175:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c64992c0 x1874672085728512/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828986.376188:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828986.376192:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787828986.376195:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828986.376200:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2421 00000100:00080000:3.0:1787828986.376205:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828986.376210:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828986.376216:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:3.0:1787828986.376230:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499e00 x1874672085728640/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828986.376240:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828986.376244:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787828986.376246:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828986.376251:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2421 00000100:00080000:3.0:1787828986.376257:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828986.376261:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787828986.376267:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085728384:192.168.203.141@tcp:38:kworker.0 00000008:00100000:3.0:1787828986.376268:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:3.0:1787828986.376302:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c64983c0 x1874672085728768/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828986.376312:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828986.376334:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:1.0:1787828986.376337:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c64992c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085728512:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1787828986.376338:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828986.376350:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2421 00000100:00080000:3.0:1787828986.376356:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828986.376360:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828986.376367:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828986.376372:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085728640:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787828986.376384:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b480 x1874672085728896/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828986.376398:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c64983c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085728768:192.168.203.141@tcp:8:kworker.0 00000100:00080000:3.0:1787828986.376399:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828986.376412:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787828986.376415:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828986.376422:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2421 00000100:00080000:3.0:1787828986.376429:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1787828986.376436:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085728896:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1787828986.376440:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787828986.376449:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828986.376476:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b0c0 x1874672085729024/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828986.376496:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828986.376500:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787828986.376502:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828986.376507:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2421 00000100:00080000:3.0:1787828986.376512:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828986.376516:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787828986.376521:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085729024:192.168.203.141@tcp:38:kworker.0 00000008:00100000:3.0:1787828986.376530:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:3.0:1787828986.376556:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a580 x1874672085729152/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828986.376576:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828986.376580:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787828986.376582:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828986.376586:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2421 00000100:00080000:3.0:1787828986.376594:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828986.376599:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787828986.376604:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085729152:192.168.203.141@tcp:8:kworker.0 00000008:00100000:3.0:1787828986.376605:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:3.0:1787828986.376632:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a1c0 x1874672085729280/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828986.376676:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085729280:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828986.503911:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828986.503911:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787828986.503957:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828986.504149:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828987.527949:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828987.527950:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828987.527954:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828987.527961:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787828988.551894:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828988.551896:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787828988.551953:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787828988.551985:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1787828989.448175:0:2396:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.141:988 00000400:00000100:2.0:1787828989.448230:0:2396:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.141@tcp at host 192.168.203.141 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1787828989.448249:0:2396:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:2.0:1787828989.448326:0:2396:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000800:00000100:2.0:1787828989.448341:0:2396:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:2.0:1787828989.448354:0:2396:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:2.0:1787828989.448372:0:2396:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:2.0:1787828989.448386:0:2396:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:2.0:1787828989.448401:0:2396:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:2.0:1787828989.448408:0:2396:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:2.0:1787828989.448419:0:2396:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:0.0:1787828989.449263:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828986/real 1787828989] req@ffff9e35c649a1c0 x1874672085729280/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829011 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828989.449309:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a1c0 x1874672085729280/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829011 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828989.449325:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828989.449348:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828989.449355:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085729280:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787828989.449375:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828986/real 1787828989] req@ffff9e35c649a580 x1874672085729152/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829011 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828989.449401:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a580 x1874672085729152/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829011 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828989.449420:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828989.449426:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828989.449445:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085729152:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787828989.449456:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828986/real 1787828989] req@ffff9e35c649b0c0 x1874672085729024/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829011 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828989.449486:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b0c0 x1874672085729024/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829011 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828989.449497:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828989.449503:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828989.449509:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085729024:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787828989.449518:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828986/real 1787828989] req@ffff9e35c649b480 x1874672085728896/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829011 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828989.449534:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b480 x1874672085728896/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829011 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828989.449551:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828989.449556:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828989.449559:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085728896:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787828989.449566:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828986/real 1787828989] req@ffff9e35c64992c0 x1874672085728512/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829011 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828989.449580:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c64992c0 x1874672085728512/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829011 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828989.449588:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828989.449592:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828989.449595:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c64992c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085728512:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787828989.449602:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828986/real 1787828989] req@ffff9e35c6499e00 x1874672085728640/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829011 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828989.449614:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499e00 x1874672085728640/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829011 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828989.449623:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828989.449627:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828989.449630:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085728640:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787828989.449638:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828986/real 1787828989] req@ffff9e35c64983c0 x1874672085728768/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829011 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828989.449652:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c64983c0 x1874672085728768/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829011 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828989.449661:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828989.449664:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828989.449667:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c64983c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085728768:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787828989.449675:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828986/real 1787828989] req@ffff9e35c649ad00 x1874672085728256/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829011 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828989.449690:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649ad00 x1874672085728256/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829011 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828989.449701:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828989.449705:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828989.449709:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085728256:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1787828989.449722:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828986/real 1787828989] req@ffff9e35c6498000 x1874672085728384/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829011 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828989.449736:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498000 x1874672085728384/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829011 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828989.449759:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828989.449764:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828989.449769:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085728384:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828989.575938:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787828989.575943:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828989.575982:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787828989.575989:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828990.599882:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828990.599891:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787828990.599893:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787828990.599978:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828991.495917:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828991.495927:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828991.495932:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828991.495943:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2426 00000100:00080000:2.0:1787828991.495958:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828991.495966:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787828991.496008:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f840 x1874672085729408/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828991.496036:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828991.496039:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828991.496041:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828991.496045:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2426 00000100:00080000:2.0:1787828991.496047:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828991.496049:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828991.496059:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828991.496070:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e580 x1874672085729536/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828991.496078:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828991.496081:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787828991.496084:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828991.496088:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2426 00000100:00080000:2.0:1787828991.496092:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828991.496104:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787828991.496106:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085729408:192.168.203.141@tcp:250:kworker.0 00000008:00100000:2.0:1787828991.496119:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828991.496151:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f0c0 x1874672085729664/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828991.496161:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828991.496165:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787828991.496166:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828991.496170:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2426 00000100:00080000:2.0:1787828991.496174:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828991.496177:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828991.496182:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000800:00000100:0.0:1787828991.496190:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828991.496197:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f480 x1874672085729792/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828991.496206:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828991.496210:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787828991.496213:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828991.496217:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2426 00000100:00080000:2.0:1787828991.496222:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828991.496226:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828991.496230:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:2.0:1787828991.496240:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e1c0 x1874672085729920/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828991.496250:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828991.496253:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828991.496255:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828991.496264:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2426 00000100:00100000:0.0:1787828991.496266:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828991/real 1787828991] req@ffff9e35c498f840 x1874672085729408/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829016 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828991.496272:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828991.496297:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828991.496309:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828991.496325:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f840 x1874672085729408/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829016 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828991.496330:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cb40 x1874672085730048/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828991.496404:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828991.496408:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787828991.496411:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828991.496413:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828991.496419:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2426 00000100:00080000:2.0:1787828991.496427:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828991.496428:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:2.0:1787828991.496438:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787828991.496443:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085729408:192.168.203.141@tcp:250:kworker.0 00000008:00100000:2.0:1787828991.496450:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828991.496461:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085729536:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828991.496476:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c000 x1874672085730176/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828991.496491:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828991.496496:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787828991.496499:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1787828991.496507:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1787828991.496511:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2426 00000100:00080000:2.0:1787828991.496521:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828991.496529:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787828991.496536:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828991/real 1787828991] req@ffff9e35c498e580 x1874672085729536/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829016 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1787828991.496537:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:2.0:1787828991.496568:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cf00 x1874672085730304/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787828991.496571:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e580 x1874672085729536/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829016 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828991.496606:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828991.496612:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787828991.496616:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828991.496617:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828991.496635:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2426 00000100:00080000:0.0:1787828991.496637:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:2.0:1787828991.496647:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787828991.496652:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085729536:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1787828991.496660:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787828991.496669:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:2.0:1787828991.496687:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498d680 x1874672085730432/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787828991.496705:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085729664:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787828991.496743:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828991.496759:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828991/real 1787828991] req@ffff9e35c498f0c0 x1874672085729664/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829016 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828991.496775:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f0c0 x1874672085729664/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829016 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828991.496792:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828991.496828:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828991.496834:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085729664:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787828991.496842:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085729792:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787828991.496866:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828991.496879:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828991/real 1787828991] req@ffff9e35c498f480 x1874672085729792/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829016 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828991.496891:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f480 x1874672085729792/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829016 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828991.496900:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828991.496904:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828991.496908:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085729792:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787828991.496915:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085729920:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787828991.496938:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828991.496949:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828991/real 1787828991] req@ffff9e35c498e1c0 x1874672085729920/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829016 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828991.496963:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e1c0 x1874672085729920/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829016 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828991.496973:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828991.496977:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828991.496981:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085729920:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787828991.496986:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085730048:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787828991.497006:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828991.497019:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828991/real 1787828991] req@ffff9e35c498cb40 x1874672085730048/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829016 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828991.497032:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498cb40 x1874672085730048/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829016 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828991.497041:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828991.497045:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828991.497048:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085730048:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787828991.497055:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085730176:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787828991.497076:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828991.497088:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828991/real 1787828991] req@ffff9e35c498c000 x1874672085730176/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829016 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828991.497100:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c000 x1874672085730176/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829016 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828991.497110:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828991.497113:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828991.497117:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085730176:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787828991.497124:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085730304:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787828991.497146:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828991.497158:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828991/real 1787828991] req@ffff9e35c498cf00 x1874672085730304/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829016 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828991.497170:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498cf00 x1874672085730304/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829016 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828991.497179:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828991.497183:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828991.497186:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085730304:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787828991.497213:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085730432:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787828991.497235:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787828991.497247:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828991/real 1787828991] req@ffff9e35c498d680 x1874672085730432/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829016 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828991.497260:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498d680 x1874672085730432/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829016 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828991.497269:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828991.497273:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787828991.497276:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085730432:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828991.623967:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787828991.623992:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828991.623993:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828991.624025:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828992.647908:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828992.647923:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787828992.647925:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:1.0:1787828992.648024:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828993.671910:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787828993.671911:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787828993.671911:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828993.671978:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828994.695872:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828994.695872:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828994.695890:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787828994.695946:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828995.719898:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787828995.719898:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787828995.719913:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787828995.719953:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828996.615909:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828996.615919:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828996.615924:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828996.615935:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2431 00000100:00080000:1.0:1787828996.615946:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828996.615954:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787828996.615989:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6d00 x1874672085730560/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828996.616023:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828996.616028:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828996.616030:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828996.616036:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2431 00000100:00080000:1.0:1787828996.616040:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828996.616044:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828996.616056:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828996.616061:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085730560:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1787828996.616075:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7480 x1874672085730688/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828996.616089:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828996.616093:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787828996.616096:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828996.616101:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2431 00000100:00080000:1.0:1787828996.616106:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828996.616110:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828996.616116:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1787828996.616126:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828996.616132:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5e00 x1874672085730816/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828996.616144:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828996.616148:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828996.616151:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828996.616157:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2431 00000100:00080000:1.0:1787828996.616162:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787828996.616163:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828996/real 1787828996] req@ffff9e35da6f6d00 x1874672085730560/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829021 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828996.616172:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828996.616180:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:2.0:1787828996.616184:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6d00 x1874672085730560/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829021 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828996.616194:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1787828996.616198:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4000 x1874672085730944/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828996.616199:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828996.616208:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085730560:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787828996.616221:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828996.616225:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828996.616229:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828996.616235:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2431 00000100:00100000:2.0:1787828996.616238:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085730688:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787828996.616255:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828996.616262:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828996.616269:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000800:00000100:2.0:1787828996.616275:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828996.616285:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4b40 x1874672085731072/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828996.616295:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828996/real 1787828996] req@ffff9e35da6f7480 x1874672085730688/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829021 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828996.616300:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828996.616308:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:2.0:1787828996.616311:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7480 x1874672085730688/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829021 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828996.616312:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828996.616323:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2431 00000100:00080000:2.0:1787828996.616324:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828996.616329:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828996.616330:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828996.616336:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085730688:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787828996.616340:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787828996.616345:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085730816:192.168.203.141@tcp:38:kworker.0 00000008:00100000:1.0:1787828996.616349:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1787828996.616365:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828996.616368:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6d00 x1874672085731200/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828996.616376:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828996/real 1787828996] req@ffff9e35da6f5e00 x1874672085730816/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829021 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828996.616389:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787828996.616394:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5e00 x1874672085730816/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829021 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828996.616395:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787828996.616400:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828996.616409:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828996.616411:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2431 00000100:00080000:2.0:1787828996.616415:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:1.0:1787828996.616419:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787828996.616420:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085730816:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787828996.616427:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787828996.616429:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085730944:192.168.203.141@tcp:8:kworker.0 00000008:00100000:1.0:1787828996.616437:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1787828996.616453:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787828996.616458:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4780 x1874672085731328/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828996.616466:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828996/real 1787828996] req@ffff9e35da6f4000 x1874672085730944/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829021 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828996.616479:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787828996.616483:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4000 x1874672085730944/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829021 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828996.616485:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828996.616490:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828996.616496:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828996.616500:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2431 00000100:00080000:2.0:1787828996.616503:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828996.616509:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085730944:192.168.203.141@tcp:8:kworker.0 00000100:00080000:1.0:1787828996.616510:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828996.616519:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787828996.616525:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:2.0:1787828996.616536:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085731072:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787828996.616543:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5a40 x1874672085731456/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787828996.616554:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787828996.616557:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828996.616563:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828996.616566:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787828996.616567:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828996/real 1787828996] req@ffff9e35da6f4b40 x1874672085731072/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829021 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828996.616576:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2431 00000100:00100000:2.0:1787828996.616582:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4b40 x1874672085731072/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829021 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828996.616583:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828996.616593:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:2.0:1787828996.616595:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828996.616600:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000008:00100000:1.0:1787828996.616601:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:2.0:1787828996.616603:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085731072:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828996.616611:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085731200:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787828996.616622:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7840 x1874672085731584/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787828996.616628:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828996.616637:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828996/real 1787828996] req@ffff9e35da6f6d00 x1874672085731200/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829021 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828996.616653:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6d00 x1874672085731200/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829021 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828996.616663:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828996.616667:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828996.616672:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085731200:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828996.616678:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085731328:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787828996.616698:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828996.616706:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828996/real 1787828996] req@ffff9e35da6f4780 x1874672085731328/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829021 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828996.616714:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4780 x1874672085731328/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829021 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828996.616720:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828996.616722:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828996.616724:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085731328:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787828996.616734:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085731456:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828996.616745:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828996.616751:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828996/real 1787828996] req@ffff9e35da6f5a40 x1874672085731456/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829021 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828996.616759:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5a40 x1874672085731456/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829021 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828996.616764:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828996.616766:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828996.616768:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085731456:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828996.616772:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085731584:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787828996.616781:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787828996.616787:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828996/real 1787828996] req@ffff9e35da6f7840 x1874672085731584/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829021 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828996.616794:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7840 x1874672085731584/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829021 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828996.616821:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828996.616823:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787828996.616826:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085731584:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787828996.743858:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787828996.743885:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787828996.743889:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787828996.743927:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828997.767924:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828997.767924:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787828997.767963:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828997.767974:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787828998.791902:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787828998.791939:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787828998.791968:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787828998.792056:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828999.815916:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787828999.815922:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787828999.815943:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787828999.815997:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829000.839950:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829000.839950:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829000.839963:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787829000.840055:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829001.735954:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829001.735966:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787829001.735973:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829001.735984:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2437 00000100:00080000:0.0:1787829001.735995:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829001.736002:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787829001.736043:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718b40 x1874672085731712/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829001.736070:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829001.736073:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829001.736074:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829001.736078:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2437 00000100:00080000:0.0:1787829001.736080:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829001.736082:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829001.736094:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829001.736112:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718f00 x1874672085731840/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829001.736125:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829001.736129:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787829001.736132:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829001.736137:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2437 00000100:00080000:0.0:1787829001.736142:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829001.736154:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787829001.736157:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085731712:192.168.203.141@tcp:250:kworker.0 00000008:00100000:0.0:1787829001.736166:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829001.736200:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a940 x1874672085731968/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829001.736227:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829001.736230:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829001.736231:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829001.736235:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2437 00000800:00000100:2.0:1787829001.736243:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787829001.736244:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829001.736251:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829001.736260:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:0.0:1787829001.736271:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7183c0 x1874672085732096/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829001.736279:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829001.736282:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829001.736283:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829001.736292:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2437 00000100:00100000:2.0:1787829001.736293:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829001/real 1787829001] req@ffff9e35da718b40 x1874672085731712/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829026 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829001.736295:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829001.736300:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829001.736306:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:0.0:1787829001.736323:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a1c0 x1874672085732224/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829001.736333:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829001.736340:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:2.0:1787829001.736341:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718b40 x1874672085731712/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829026 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829001.736344:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829001.736357:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2437 00000100:00080000:0.0:1787829001.736363:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829001.736369:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829001.736373:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1787829001.736376:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1787829001.736382:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718780 x1874672085732352/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829001.736389:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:0.0:1787829001.736393:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829001.736399:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787829001.736402:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787829001.736405:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085731712:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1787829001.736407:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2437 00000100:00080000:0.0:1787829001.736417:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829001.736425:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829001.736431:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829001.736434:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085731840:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829001.736448:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da719a40 x1874672085732480/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829001.736462:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829001.736467:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829001.736470:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829001.736478:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2437 00000100:00080000:0.0:1787829001.736483:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1787829001.736486:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787829001.736492:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829001.736501:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:2.0:1787829001.736510:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829001/real 1787829001] req@ffff9e35da718f00 x1874672085731840/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829026 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829001.736529:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a580 x1874672085732608/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829001.736535:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672085731840/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829026 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829001.736544:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829001.736554:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829001.736559:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829001.736567:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2437 00000100:00080000:0.0:1787829001.736574:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829001.736576:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829001.736579:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829001.736585:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00080000:2.0:1787829001.736587:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829001.736598:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085731840:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829001.736605:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718000 x1874672085732736/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829001.736642:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085731968:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787829001.736671:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829001.736685:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829001/real 1787829001] req@ffff9e35da71a940 x1874672085731968/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829026 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829001.736700:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672085731968/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829026 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829001.736709:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829001.736713:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829001.736717:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085731968:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829001.736724:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085732096:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829001.736745:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829001.736757:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829001/real 1787829001] req@ffff9e35da7183c0 x1874672085732096/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829026 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829001.736771:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672085732096/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829026 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829001.736781:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829001.736784:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829001.736788:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085732096:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829001.736795:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085732224:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829001.736860:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829001.736872:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829001/real 1787829001] req@ffff9e35da71a1c0 x1874672085732224/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829026 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829001.736885:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a1c0 x1874672085732224/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829026 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829001.736894:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829001.736898:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829001.736902:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085732224:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829001.736908:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085732352:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787829001.736930:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829001.736942:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829001/real 1787829001] req@ffff9e35da718780 x1874672085732352/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829026 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829001.736955:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718780 x1874672085732352/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829026 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829001.736971:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829001.736974:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829001.736977:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085732352:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829001.736984:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085732480:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787829001.737015:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829001.737028:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829001/real 1787829001] req@ffff9e35da719a40 x1874672085732480/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829026 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829001.737040:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672085732480/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829026 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829001.737048:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829001.737051:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829001.737053:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085732480:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829001.737059:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085732608:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829001.737081:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829001.737093:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829001/real 1787829001] req@ffff9e35da71a580 x1874672085732608/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829026 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829001.737106:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672085732608/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829026 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829001.737115:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829001.737118:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829001.737122:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085732608:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829001.737129:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085732736:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829001.737148:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829001.737156:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829001/real 1787829001] req@ffff9e35da718000 x1874672085732736/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829026 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829001.737165:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718000 x1874672085732736/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829026 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829001.737171:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829001.737173:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829001.737175:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085732736:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829001.863913:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829001.863915:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829001.863921:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787829001.864017:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829002.887916:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829002.887916:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829002.887972:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787829002.887973:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829003.911886:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829003.911886:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829003.911887:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787829003.911946:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829004.935915:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829004.935915:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829004.935915:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787829004.936064:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829005.959883:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829005.959884:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829005.959884:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787829005.959974:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829006.855911:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829006.855925:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787829006.855929:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829006.855937:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2442 00000100:00080000:3.0:1787829006.855946:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829006.855952:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787829006.855989:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498000 x1874672085732864/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829006.856008:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829006.856011:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829006.856012:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829006.856015:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2442 00000100:00080000:3.0:1787829006.856017:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829006.856019:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829006.856027:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829006.856035:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649ad00 x1874672085732992/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829006.856042:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829006.856045:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787829006.856046:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829006.856049:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2442 00000100:00080000:3.0:1787829006.856051:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829006.856054:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829006.856058:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829006.856066:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c64983c0 x1874672085733120/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829006.856068:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085732864:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1787829006.856082:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829006.856084:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787829006.856086:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829006.856088:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2442 00000100:00080000:3.0:1787829006.856090:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829006.856092:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829006.856095:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:3.0:1787829006.856102:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499e00 x1874672085733248/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829006.856108:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829006.856110:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787829006.856112:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829006.856114:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2442 00000100:00080000:3.0:1787829006.856127:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829006.856130:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829006.856133:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000800:00000100:2.0:1787829006.856144:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787829006.856151:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c64992c0 x1874672085733376/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829006.856160:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829006.856163:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829006.856164:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829006.856168:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2442 00000100:00080000:3.0:1787829006.856171:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829006.856185:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829006.856189:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829006.856199:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b480 x1874672085733504/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829006.856206:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787829006.856208:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829006/real 1787829006] req@ffff9e35c6498000 x1874672085732864/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829031 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829006.856210:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787829006.856213:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829006.856221:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2442 00000100:00080000:3.0:1787829006.856227:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829006.856232:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829006.856238:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829006.856252:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b0c0 x1874672085733632/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829006.856260:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498000 x1874672085732864/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829031 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829006.856267:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829006.856279:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787829006.856282:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829006.856283:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829006.856289:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2442 00000100:00080000:2.0:1787829006.856292:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:3.0:1787829006.856297:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787829006.856300:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085732864:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1787829006.856307:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787829006.856315:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085732992:192.168.203.141@tcp:38:kworker.0 00000008:00100000:3.0:1787829006.856316:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:3.0:1787829006.856335:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a580 x1874672085733760/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829006.856347:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1787829006.856351:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1787829006.856355:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787829006.856358:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829006.856365:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2442 00000100:00080000:3.0:1787829006.856369:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787829006.856373:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829006/real 1787829006] req@ffff9e35c649ad00 x1874672085732992/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829031 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829006.856378:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787829006.856395:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649ad00 x1874672085732992/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829031 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1787829006.856395:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00080000:2.0:1787829006.856410:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1787829006.856415:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a1c0 x1874672085733888/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829006.856418:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829006.856427:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085732992:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829006.856459:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c64983c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085733120:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787829006.856482:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829006.856495:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829006/real 1787829006] req@ffff9e35c64983c0 x1874672085733120/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829031 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829006.856508:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c64983c0 x1874672085733120/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829031 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829006.856524:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829006.856528:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829006.856531:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c64983c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085733120:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829006.856538:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085733248:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829006.856559:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829006.856571:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829006/real 1787829006] req@ffff9e35c6499e00 x1874672085733248/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829031 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829006.856583:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499e00 x1874672085733248/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829031 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829006.856592:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829006.856595:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829006.856598:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085733248:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829006.856604:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c64992c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085733376:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829006.856624:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829006.856635:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829006/real 1787829006] req@ffff9e35c64992c0 x1874672085733376/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829031 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829006.856647:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c64992c0 x1874672085733376/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829031 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829006.856656:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829006.856659:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829006.856662:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c64992c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085733376:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829006.856669:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085733504:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787829006.856689:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829006.856701:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829006/real 1787829006] req@ffff9e35c649b480 x1874672085733504/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829031 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829006.856713:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b480 x1874672085733504/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829031 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829006.856722:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829006.856725:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829006.856729:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085733504:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829006.856735:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085733632:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787829006.856755:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829006.856767:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829006/real 1787829006] req@ffff9e35c649b0c0 x1874672085733632/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829031 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829006.856780:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b0c0 x1874672085733632/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829031 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829006.856789:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829006.856792:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829006.856836:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085733632:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829006.856844:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085733760:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829006.856866:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829006.856879:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829006/real 1787829006] req@ffff9e35c649a580 x1874672085733760/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829031 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829006.856891:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a580 x1874672085733760/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829031 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829006.856901:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829006.856904:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829006.856908:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085733760:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829006.856922:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085733888:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829006.856942:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829006.856955:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829006/real 1787829006] req@ffff9e35c649a1c0 x1874672085733888/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829031 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829006.856967:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a1c0 x1874672085733888/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829031 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829006.856976:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829006.856980:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829006.856984:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085733888:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829006.983849:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829006.983860:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829006.983875:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787829006.983956:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829008.007877:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:3.0:1787829008.007882:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829008.007885:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787829008.007977:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829009.031918:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829009.031920:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829009.031935:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787829009.032037:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829010.055891:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:3.0:1787829010.055899:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829010.055903:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787829010.055990:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829011.079919:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829011.079919:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:3.0:1787829011.079927:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787829011.080056:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829011.975981:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829011.975996:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787829011.976001:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829011.976015:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2447 00000100:00080000:0.0:1787829011.976030:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829011.976039:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787829011.976093:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7192c0 x1874672085734016/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829011.976152:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829011.976157:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829011.976159:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829011.976163:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2447 00000100:00080000:0.0:1787829011.976167:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829011.976171:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829011.976184:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829011.976220:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718000 x1874672085734144/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829011.976236:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829011.976240:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787829011.976243:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829011.976247:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2447 00000100:00080000:0.0:1787829011.976250:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829011.976254:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829011.976260:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829011.976272:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a580 x1874672085734272/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829011.976287:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829011.976293:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:2.0:1787829011.976293:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085734016:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1787829011.976299:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829011.976315:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2447 00000100:00080000:0.0:1787829011.976327:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829011.976333:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829011.976338:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:0.0:1787829011.976352:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da719a40 x1874672085734400/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829011.976380:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829011.976384:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829011.976387:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829011.976393:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2447 00000100:00080000:0.0:1787829011.976401:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829011.976406:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829011.976412:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:0.0:1787829011.976428:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718780 x1874672085734528/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829011.976449:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787829011.976468:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829011.976473:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829011.976475:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829011.976482:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2447 00000100:00080000:0.0:1787829011.976487:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829011.976499:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829011.976505:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829011.976520:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a1c0 x1874672085734656/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829011.976537:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829011.976542:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787829011.976543:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829011.976548:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2447 00000100:00080000:0.0:1787829011.976550:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829011.976553:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829011.976560:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829011.976561:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829011/real 1787829011] req@ffff9e35da7192c0 x1874672085734016/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829036 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829011.976579:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7183c0 x1874672085734784/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829011.976638:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829011.976643:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829011.976645:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829011.976649:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2447 00000100:00080000:0.0:1787829011.976653:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829011.976658:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787829011.976672:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7192c0 x1874672085734016/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829036 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1787829011.976673:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:0.0:1787829011.976695:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a940 x1874672085734912/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829011.976722:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829011.976723:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829011.976747:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829011.976755:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:0.0:1787829011.976756:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829011.976775:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2447 00000100:00100000:2.0:1787829011.976777:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085734016:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1787829011.976794:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829011.976848:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829011.976854:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:2.0:1787829011.976864:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085734144:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829011.976877:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718f00 x1874672085735040/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829011.976927:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829011.976948:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829011/real 1787829011] req@ffff9e35da718000 x1874672085734144/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829036 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829011.976973:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718000 x1874672085734144/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829036 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829011.976985:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829011.976990:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829011.976996:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085734144:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829011.977053:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085734272:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787829011.977095:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829011.977121:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829011/real 1787829011] req@ffff9e35da71a580 x1874672085734272/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829036 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829011.977161:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672085734272/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829036 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829011.977175:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829011.977180:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829011.977184:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085734272:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829011.977193:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085734400:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829011.977222:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829011.977246:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829011/real 1787829011] req@ffff9e35da719a40 x1874672085734400/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829036 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829011.977260:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672085734400/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829036 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829011.977270:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829011.977274:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829011.977278:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085734400:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829011.977289:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085734528:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829011.977323:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829011.977343:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829011/real 1787829011] req@ffff9e35da718780 x1874672085734528/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829036 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829011.977368:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718780 x1874672085734528/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829036 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829011.977385:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829011.977390:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829011.977394:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085734528:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829011.977402:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085734656:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787829011.977429:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829011.977442:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829011/real 1787829011] req@ffff9e35da71a1c0 x1874672085734656/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829036 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829011.977474:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a1c0 x1874672085734656/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829036 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829011.977489:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829011.977495:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829011.977502:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085734656:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829011.977514:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085734784:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787829011.977552:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829011.977570:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829011/real 1787829011] req@ffff9e35da7183c0 x1874672085734784/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829036 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829011.977585:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672085734784/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829036 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829011.977595:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829011.977598:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829011.977603:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085734784:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829011.977611:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085734912:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829011.977632:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829011.977643:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829011/real 1787829011] req@ffff9e35da71a940 x1874672085734912/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829036 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829011.977656:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672085734912/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829036 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829011.977666:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829011.977669:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829011.977672:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085734912:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829011.977678:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085735040:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829011.977695:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829011.977705:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829011/real 1787829011] req@ffff9e35da718f00 x1874672085735040/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829036 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829011.977718:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672085735040/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829036 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829011.977727:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829011.977730:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829011.977734:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085735040:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829012.103889:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829012.103934:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:3.0:1787829012.103934:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787829012.104047:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829013.127937:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829013.127937:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:3.0:1787829013.127937:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787829013.128039:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829014.151908:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829014.151910:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:3.0:1787829014.151925:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787829014.152043:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829015.175957:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829015.175957:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:3.0:1787829015.175957:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787829015.176081:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829016.199948:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829016.199958:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829016.200011:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787829016.200056:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829017.095916:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829017.095923:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787829017.095928:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829017.095936:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2452 00000100:00080000:0.0:1787829017.095945:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829017.095951:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787829017.095984:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718b40 x1874672085735168/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829017.096007:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829017.096010:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829017.096012:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829017.096014:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2452 00000100:00080000:0.0:1787829017.096017:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829017.096030:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829017.096038:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829017.096048:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718f00 x1874672085735296/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829017.096055:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829017.096058:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787829017.096060:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829017.096062:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2452 00000100:00080000:0.0:1787829017.096065:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829017.096068:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829017.096072:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829017.096083:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a940 x1874672085735424/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829017.096094:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829017.096097:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829017.096100:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829017.096116:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2452 00000100:00080000:0.0:1787829017.096120:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829017.096125:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829017.096131:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:0.0:1787829017.096146:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7183c0 x1874672085735552/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829017.096167:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829017.096172:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829017.096175:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829017.096182:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2452 00000100:00080000:0.0:1787829017.096188:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829017.096193:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829017.096199:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:0.0:1787829017.096217:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a1c0 x1874672085735680/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829017.096235:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085735168:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1787829017.096236:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829017.096257:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829017.096259:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829017.096266:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2452 00000100:00080000:0.0:1787829017.096271:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829017.096277:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829017.096285:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829017.096301:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718780 x1874672085735808/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829017.096309:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829017.096312:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787829017.096313:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829017.096317:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2452 00000100:00080000:0.0:1787829017.096321:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829017.096335:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829017.096339:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829017.096347:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da719a40 x1874672085735936/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829017.096353:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829017.096356:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829017.096358:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829017.096362:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2452 00000100:00080000:0.0:1787829017.096365:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829017.096368:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829017.096371:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:0.0:1787829017.096385:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a580 x1874672085736064/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829017.096389:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787829017.096405:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829017.096410:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829017.096411:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829017.096416:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2452 00000100:00080000:0.0:1787829017.096420:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829017.096424:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829017.096428:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:0.0:1787829017.096437:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718000 x1874672085736192/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829017.096497:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829017/real 1787829017] req@ffff9e35da718b40 x1874672085735168/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829042 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829017.096556:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718b40 x1874672085735168/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829042 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829017.096578:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829017.096584:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829017.096589:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085735168:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1787829017.096601:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085735296:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787829017.096644:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829017.096670:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829017/real 1787829017] req@ffff9e35da718f00 x1874672085735296/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829042 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829017.096719:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672085735296/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829042 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829017.096736:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829017.096742:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829017.096750:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085735296:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829017.096763:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085735424:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787829017.096860:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829017.096880:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829017/real 1787829017] req@ffff9e35da71a940 x1874672085735424/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829042 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829017.096895:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672085735424/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829042 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829017.096906:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829017.096911:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829017.096918:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085735424:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829017.096928:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085735552:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829017.096954:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829017.096965:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829017/real 1787829017] req@ffff9e35da7183c0 x1874672085735552/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829042 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829017.096978:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672085735552/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829042 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829017.097009:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829017.097012:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829017.097016:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085735552:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829017.097061:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085735680:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829017.097086:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829017.097143:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829017/real 1787829017] req@ffff9e35da71a1c0 x1874672085735680/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829042 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829017.097169:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a1c0 x1874672085735680/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829042 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829017.097183:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829017.097189:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829017.097194:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085735680:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829017.097205:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085735808:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787829017.097242:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829017.097264:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829017/real 1787829017] req@ffff9e35da718780 x1874672085735808/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829042 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829017.097283:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718780 x1874672085735808/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829042 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829017.097297:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829017.097302:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829017.097308:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085735808:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829017.097318:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085735936:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787829017.097372:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829017.097395:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829017/real 1787829017] req@ffff9e35da719a40 x1874672085735936/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829042 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829017.097419:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672085735936/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829042 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829017.097437:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829017.097443:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829017.097450:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085735936:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829017.097462:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085736064:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829017.097493:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829017.097509:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829017/real 1787829017] req@ffff9e35da71a580 x1874672085736064/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829042 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829017.097524:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672085736064/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829042 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829017.097535:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829017.097539:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829017.097543:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085736064:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829017.097568:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085736192:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829017.097590:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829017.097601:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829017/real 1787829017] req@ffff9e35da718000 x1874672085736192/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829042 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829017.097615:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718000 x1874672085736192/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829042 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829017.097624:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829017.097627:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829017.097632:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085736192:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829017.223913:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829017.223930:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829017.224007:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829017.224009:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829018.248028:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829018.248076:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829018.248080:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829018.248131:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829019.272003:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829019.272057:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:3.0:1787829019.272077:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787829019.272290:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829020.295874:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829020.295933:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829020.295935:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787829020.295963:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829021.319922:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829021.319930:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829021.319964:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829021.320005:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829022.215927:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829022.215938:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829022.215953:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829022.215963:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2457 00000100:00080000:1.0:1787829022.215975:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829022.215982:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787829022.216040:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4f00 x1874672085736320/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829022.216082:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829022.216086:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829022.216088:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829022.216091:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2457 00000100:00080000:1.0:1787829022.216105:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829022.216109:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829022.216120:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829022.216133:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f61c0 x1874672085736448/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829022.216136:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085736320:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787829022.216151:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829022.216154:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829022.216155:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829022.216170:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2457 00000100:00080000:1.0:1787829022.216173:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829022.216175:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829022.216178:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829022.216186:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f43c0 x1874672085736576/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829022.216193:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829022.216195:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829022.216197:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829022.216199:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2457 00000100:00080000:1.0:1787829022.216203:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829022.216206:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829022.216210:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:1.0:1787829022.216219:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6580 x1874672085736704/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829022.216227:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787829022.216229:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829022.216233:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829022.216244:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829022.216248:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2457 00000100:00080000:1.0:1787829022.216251:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829022.216253:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829022.216257:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:1.0:1787829022.216265:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6940 x1874672085736832/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829022.216271:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829022.216274:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829022.216276:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829022.216279:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2457 00000100:00080000:1.0:1787829022.216282:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829022.216285:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787829022.216289:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829022/real 1787829022] req@ffff9e35da6f4f00 x1874672085736320/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829047 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1787829022.216290:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829022.216302:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f70c0 x1874672085736960/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829022.216319:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787829022.216322:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4f00 x1874672085736320/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829047 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829022.216323:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829022.216327:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829022.216335:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2457 00000100:00080000:1.0:1787829022.216338:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829022.216339:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829022.216341:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:2.0:1787829022.216345:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000008:00100000:1.0:1787829022.216346:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829022.216352:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085736320:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1787829022.216359:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7840 x1874672085737088/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829022.216366:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829022.216369:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829022.216370:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829022.216373:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2457 00000100:00080000:1.0:1787829022.216375:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829022.216378:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829022.216381:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:2.0:1787829022.216382:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085736448:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829022.216391:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5a40 x1874672085737216/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829022.216399:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829022.216402:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829022.216403:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1787829022.216406:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787829022.216410:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2457 00000100:00080000:1.0:1787829022.216425:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829022.216429:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829022.216433:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:2.0:1787829022.216437:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829022/real 1787829022] req@ffff9e35da6f61c0 x1874672085736448/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829047 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829022.216445:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4780 x1874672085737344/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829022.216451:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f61c0 x1874672085736448/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829047 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829022.216463:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829022.216468:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829022.216470:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085736448:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829022.216474:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085736576:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787829022.216488:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829022.216496:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829022/real 1787829022] req@ffff9e35da6f43c0 x1874672085736576/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829047 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829022.216509:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f43c0 x1874672085736576/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829047 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829022.216517:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829022.216520:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829022.216524:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085736576:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829022.216530:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085736704:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829022.216560:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829022.216568:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829022/real 1787829022] req@ffff9e35da6f6580 x1874672085736704/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829047 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829022.216576:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6580 x1874672085736704/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829047 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829022.216582:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829022.216584:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829022.216586:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085736704:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829022.216589:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085736832:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829022.216601:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829022.216607:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829022/real 1787829022] req@ffff9e35da6f6940 x1874672085736832/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829047 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829022.216625:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6940 x1874672085736832/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829047 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829022.216631:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829022.216633:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829022.216634:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085736832:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829022.216638:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085736960:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787829022.216648:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829022.216654:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829022/real 1787829022] req@ffff9e35da6f70c0 x1874672085736960/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829047 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829022.216661:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f70c0 x1874672085736960/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829047 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829022.216666:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829022.216668:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829022.216669:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085736960:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829022.216673:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085737088:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787829022.216683:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829022.216689:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829022/real 1787829022] req@ffff9e35da6f7840 x1874672085737088/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829047 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829022.216711:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7840 x1874672085737088/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829047 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829022.216716:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829022.216718:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829022.216720:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085737088:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829022.216738:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085737216:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829022.216749:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829022.216755:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829022/real 1787829022] req@ffff9e35da6f5a40 x1874672085737216/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829047 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829022.216762:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5a40 x1874672085737216/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829047 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829022.216780:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829022.216782:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829022.216785:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085737216:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829022.216791:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085737344:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829022.216846:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829022.216856:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829022/real 1787829022] req@ffff9e35da6f4780 x1874672085737344/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829047 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829022.216864:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4780 x1874672085737344/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829047 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829022.216880:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829022.216884:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829022.216888:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085737344:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829022.343956:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829022.343956:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829022.343983:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787829022.344102:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829023.367878:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829023.367878:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829023.367894:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829023.367898:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829024.391916:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829024.391965:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829024.391970:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829024.391987:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829025.415906:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829025.415906:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829025.415942:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829025.415953:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829026.439886:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829026.439955:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829026.439969:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829026.439970:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787829027.335919:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829027.335930:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787829027.335934:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829027.335945:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2462 00000100:00080000:0.0:1787829027.335958:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829027.335964:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787829027.336010:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7192c0 x1874672085737472/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829027.336052:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829027.336056:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829027.336057:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829027.336061:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2462 00000100:00080000:0.0:1787829027.336064:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829027.336067:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829027.336078:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829027.336094:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718000 x1874672085737600/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829027.336104:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085737472:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1787829027.336113:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829027.336118:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787829027.336120:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829027.336126:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2462 00000100:00080000:0.0:1787829027.336131:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829027.336136:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829027.336142:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829027.336157:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a580 x1874672085737728/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829027.336169:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829027.336173:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829027.336175:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1787829027.336179:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787829027.336183:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2462 00000100:00080000:0.0:1787829027.336188:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829027.336192:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829027.336198:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:0.0:1787829027.336208:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da719a40 x1874672085737856/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829027.336216:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829027.336219:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829027.336220:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829027.336223:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2462 00000100:00080000:0.0:1787829027.336226:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829027.336229:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829027.336232:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:2.0:1787829027.336240:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829027/real 1787829027] req@ffff9e35da7192c0 x1874672085737472/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829052 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829027.336242:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718780 x1874672085737984/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829027.336271:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829027.336274:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829027.336276:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829027.336279:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2462 00000100:00080000:0.0:1787829027.336282:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787829027.336283:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7192c0 x1874672085737472/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829052 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829027.336287:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829027.336295:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1787829027.336301:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1787829027.336306:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a1c0 x1874672085738112/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829027.336308:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829027.336316:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085737472:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1787829027.336323:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829027.336326:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787829027.336328:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829027.336331:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2462 00000100:00080000:0.0:1787829027.336333:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829027.336336:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829027.336339:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829027.336347:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7183c0 x1874672085738240/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829027.336350:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085737600:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787829027.336363:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829027.336366:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829027.336367:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1787829027.336379:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787829027.336380:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2462 00000100:00080000:0.0:1787829027.336387:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829027.336390:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829027.336394:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:2.0:1787829027.336397:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829027/real 1787829027] req@ffff9e35da718000 x1874672085737600/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829052 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829027.336407:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a940 x1874672085738368/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829027.336412:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718000 x1874672085737600/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829052 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829027.336419:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829027.336426:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829027.336430:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829027.336431:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829027.336436:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2462 00000100:00080000:2.0:1787829027.336439:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:0.0:1787829027.336441:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787829027.336445:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085737600:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787829027.336449:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829027.336454:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:2.0:1787829027.336456:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085737728:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829027.336464:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718f00 x1874672085738496/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829027.336475:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829027.336485:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829027/real 1787829027] req@ffff9e35da71a580 x1874672085737728/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829052 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829027.336493:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672085737728/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829052 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829027.336500:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829027.336503:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829027.336506:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085737728:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829027.336510:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085737856:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829027.336521:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829027.336528:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829027/real 1787829027] req@ffff9e35da719a40 x1874672085737856/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829052 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829027.336535:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672085737856/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829052 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829027.336545:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829027.336547:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829027.336549:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085737856:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829027.336555:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085737984:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829027.336581:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829027.336594:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829027/real 1787829027] req@ffff9e35da718780 x1874672085737984/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829052 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829027.336607:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718780 x1874672085737984/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829052 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829027.336614:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829027.336616:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829027.336619:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085737984:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829027.336623:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085738112:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787829027.336639:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829027.336646:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829027/real 1787829027] req@ffff9e35da71a1c0 x1874672085738112/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829052 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829027.336654:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a1c0 x1874672085738112/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829052 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829027.336659:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829027.336661:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829027.336664:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085738112:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829027.336680:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085738240:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787829027.336692:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829027.336699:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829027/real 1787829027] req@ffff9e35da7183c0 x1874672085738240/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829052 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829027.336706:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672085738240/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829052 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829027.336712:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829027.336713:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829027.336715:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085738240:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829027.336718:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085738368:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829027.336728:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829027.336734:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829027/real 1787829027] req@ffff9e35da71a940 x1874672085738368/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829052 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829027.336741:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672085738368/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829052 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829027.336758:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829027.336761:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829027.336764:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085738368:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829027.336771:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085738496:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829027.336791:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829027.336828:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829027/real 1787829027] req@ffff9e35da718f00 x1874672085738496/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829052 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829027.336837:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672085738496/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829052 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829027.336844:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829027.336846:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829027.336848:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085738496:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829027.463894:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829027.463911:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829027.464017:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829027.464037:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829028.487910:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787829028.487986:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829028.487995:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:3.0:1787829028.488010:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829029.511915:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:3.0:1787829029.511916:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829029.511962:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829029.512035:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829030.535902:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829030.535903:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829030.535921:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829030.535957:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829031.559902:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829031.559902:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787829031.559936:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829031.559946:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00080000:0.0:1787829032.455892:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829032.455923:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787829032.455948:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829032.455957:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2467 00000100:00080000:0.0:1787829032.455966:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829032.455972:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787829032.456041:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718b40 x1874672085738624/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829032.456062:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829032.456065:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829032.456067:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829032.456071:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2467 00000100:00080000:0.0:1787829032.456090:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829032.456095:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829032.456104:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829032.456118:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085738624:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1787829032.456120:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718f00 x1874672085738752/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829032.456135:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829032.456137:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787829032.456139:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829032.456143:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2467 00000100:00080000:0.0:1787829032.456145:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829032.456148:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829032.456171:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829032.456183:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a940 x1874672085738880/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829032.456191:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829032.456193:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829032.456195:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829032.456200:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2467 00000800:00000100:2.0:1787829032.456202:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787829032.456204:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829032.456209:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829032.456213:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:0.0:1787829032.456222:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7183c0 x1874672085739008/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829032.456228:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829032.456232:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829032.456233:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829032.456238:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2467 00000100:00080000:0.0:1787829032.456241:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829032.456246:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787829032.456246:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829032/real 1787829032] req@ffff9e35da718b40 x1874672085738624/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829057 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1787829032.456252:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:0.0:1787829032.456264:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a1c0 x1874672085739136/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829032.456270:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718b40 x1874672085738624/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829057 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829032.456274:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829032.456286:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829032.456289:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829032.456292:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085738624:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1787829032.456302:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829032.456305:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829032.456309:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2467 00000100:00080000:0.0:1787829032.456312:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829032.456315:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787829032.456319:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085738752:192.168.203.141@tcp:38:kworker.0 00000008:00100000:0.0:1787829032.456321:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829032.456335:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718780 x1874672085739264/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829032.456343:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1787829032.456346:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787829032.456347:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787829032.456350:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829032.456355:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2467 00000100:00080000:0.0:1787829032.456358:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787829032.456362:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829032/real 1787829032] req@ffff9e35da718f00 x1874672085738752/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829057 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829032.456364:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829032.456371:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829032.456383:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672085738752/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829057 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829032.456385:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da719a40 x1874672085739392/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829032.456397:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829032.456400:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829032.456402:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085738752:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829032.456406:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085738880:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787829032.456422:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787829032.456429:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787829032.456432:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829032/real 1787829032] req@ffff9e35da71a940 x1874672085738880/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829057 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829032.456434:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829032.456437:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829032.456458:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2467 00000100:00080000:0.0:1787829032.456462:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787829032.456462:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672085738880/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829057 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829032.456467:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829032.456476:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00080000:2.0:1787829032.456478:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829032.456483:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829032.456485:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085738880:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829032.456489:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085739008:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829032.456502:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829032.456508:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829032/real 1787829032] req@ffff9e35da7183c0 x1874672085739008/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829057 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829032.456516:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a580 x1874672085739520/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829032.456521:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672085739008/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829057 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829032.456544:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829032.456557:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829032.456559:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085739008:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829032.456562:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085739136:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829032.456572:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787829032.456573:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787829032.456580:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829032/real 1787829032] req@ffff9e35da71a1c0 x1874672085739136/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829057 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829032.456581:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:2.0:1787829032.456590:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a1c0 x1874672085739136/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829057 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829032.456596:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829032.456598:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:0.0:1787829032.456601:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787829032.456602:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085739136:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829032.456618:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085739264:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787829032.456624:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2467 00000800:00000100:2.0:1787829032.456633:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787829032.456639:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787829032.456642:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829032/real 1787829032] req@ffff9e35da718780 x1874672085739264/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829057 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829032.456647:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829032.456657:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:2.0:1787829032.456661:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718780 x1874672085739264/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829057 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829032.456674:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829032.456679:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829032.456683:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718000 x1874672085739648/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829032.456685:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085739264:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829032.456695:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085739392:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787829032.456719:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829032.456738:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829032/real 1787829032] req@ffff9e35da719a40 x1874672085739392/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829057 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829032.456747:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672085739392/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829057 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829032.456753:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829032.456756:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829032.456759:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085739392:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829032.456764:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085739520:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829032.456780:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829032.456786:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829032/real 1787829032] req@ffff9e35da71a580 x1874672085739520/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829057 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829032.456794:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672085739520/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829057 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829032.456828:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829032.456831:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829032.456835:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085739520:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829032.456860:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085739648:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829032.456875:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829032.456887:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829032/real 1787829032] req@ffff9e35da718000 x1874672085739648/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829057 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829032.456897:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718000 x1874672085739648/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829057 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829032.456905:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829032.456907:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829032.456911:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085739648:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829032.583895:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829032.583904:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829032.583914:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829032.584010:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829033.607920:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829033.607922:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829033.607923:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829033.607955:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829034.631909:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829034.631910:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829034.631944:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829034.631968:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829035.655892:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829035.655894:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829035.655896:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829035.655927:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829036.679877:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829036.679877:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:3.0:1787829036.679879:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829036.679942:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829037.575889:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829037.575897:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787829037.575901:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829037.575909:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2472 00000100:00080000:0.0:1787829037.575918:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829037.575923:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787829037.575957:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7192c0 x1874672085739776/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829037.575997:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829037.575999:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829037.576001:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829037.576004:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2472 00000100:00080000:0.0:1787829037.576017:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829037.576020:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829037.576028:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829037.576041:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718000 x1874672085739904/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829037.576050:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829037.576052:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787829037.576054:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829037.576058:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2472 00000100:00100000:2.0:1787829037.576061:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085739776:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1787829037.576062:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829037.576066:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829037.576072:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829037.576080:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a580 x1874672085740032/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829037.576091:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829037.576094:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829037.576095:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829037.576098:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2472 00000100:00080000:0.0:1787829037.576101:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829037.576103:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829037.576106:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:0.0:1787829037.576114:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da719a40 x1874672085740160/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829037.576120:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829037.576122:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829037.576124:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829037.576126:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2472 00000100:00080000:0.0:1787829037.576131:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1787829037.576131:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787829037.576137:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829037.576140:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:0.0:1787829037.576148:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718780 x1874672085740288/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829037.576155:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829037.576157:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829037.576158:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829037.576160:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2472 00000100:00080000:0.0:1787829037.576164:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829037.576167:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829037.576171:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829037.576178:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829037/real 1787829037] req@ffff9e35da7192c0 x1874672085739776/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829062 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829037.576180:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a1c0 x1874672085740416/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829037.576208:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829037.576211:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787829037.576212:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829037.576216:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2472 00000100:00080000:0.0:1787829037.576220:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787829037.576220:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7192c0 x1874672085739776/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829062 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829037.576231:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829037.576234:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:2.0:1787829037.576238:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000008:00100000:0.0:1787829037.576241:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829037.576245:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085739776:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1787829037.576252:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7183c0 x1874672085740544/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829037.576261:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829037.576263:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829037.576264:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829037.576267:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2472 00000100:00080000:0.0:1787829037.576270:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787829037.576272:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085739904:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787829037.576276:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829037.576281:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:0.0:1787829037.576289:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a940 x1874672085740672/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829037.576300:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787829037.576309:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829037.576312:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829037.576313:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787829037.576317:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829037/real 1787829037] req@ffff9e35da718000 x1874672085739904/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829062 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829037.576328:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2472 00000100:00080000:0.0:1787829037.576333:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829037.576339:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787829037.576343:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718000 x1874672085739904/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829062 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1787829037.576344:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00080000:2.0:1787829037.576356:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1787829037.576359:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718f00 x1874672085740800/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829037.576362:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829037.576369:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085739904:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829037.576380:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085740032:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787829037.576395:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829037.576402:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829037/real 1787829037] req@ffff9e35da71a580 x1874672085740032/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829062 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829037.576419:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672085740032/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829062 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829037.576435:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829037.576437:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829037.576440:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085740032:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829037.576444:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085740160:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829037.576455:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829037.576462:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829037/real 1787829037] req@ffff9e35da719a40 x1874672085740160/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829062 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829037.576475:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672085740160/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829062 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829037.576482:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829037.576484:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829037.576486:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085740160:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829037.576489:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085740288:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829037.576500:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829037.576506:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829037/real 1787829037] req@ffff9e35da718780 x1874672085740288/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829062 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829037.576513:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718780 x1874672085740288/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829062 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829037.576519:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829037.576520:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829037.576522:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085740288:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829037.576526:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085740416:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787829037.576536:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829037.576542:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829037/real 1787829037] req@ffff9e35da71a1c0 x1874672085740416/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829062 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829037.576549:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a1c0 x1874672085740416/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829062 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829037.576555:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829037.576557:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829037.576559:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085740416:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829037.576562:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085740544:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787829037.576571:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829037.576577:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829037/real 1787829037] req@ffff9e35da7183c0 x1874672085740544/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829062 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829037.576583:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672085740544/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829062 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829037.576589:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829037.576590:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829037.576592:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085740544:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829037.576607:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085740672:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829037.576617:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829037.576624:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829037/real 1787829037] req@ffff9e35da71a940 x1874672085740672/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829062 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829037.576631:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672085740672/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829062 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829037.576643:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829037.576645:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829037.576647:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085740672:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829037.576662:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085740800:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829037.576673:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829037.576679:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829037/real 1787829037] req@ffff9e35da718f00 x1874672085740800/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829062 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829037.576687:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672085740800/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829062 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829037.576693:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829037.576694:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829037.576696:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085740800:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829037.703905:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829037.703905:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:0.0:1787829037.703954:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829037.703956:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829038.727907:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829038.727965:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829038.727987:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829038.727989:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829039.751957:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829039.751965:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829039.751976:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829039.752008:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829040.775895:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829040.775895:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829040.775914:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:0.0:1787829040.775982:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829041.799906:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829041.799906:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787829041.799959:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829041.799965:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787829042.695916:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829042.695938:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787829042.695943:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829042.695954:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2478 00000100:00080000:0.0:1787829042.695965:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829042.695973:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787829042.696005:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718b40 x1874672085740928/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829042.696031:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829042.696035:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829042.696038:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829042.696043:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2478 00000100:00080000:0.0:1787829042.696046:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829042.696050:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829042.696061:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829042.696074:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085740928:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1787829042.696078:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718f00 x1874672085741056/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829042.696092:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829042.696096:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787829042.696098:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829042.696103:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2478 00000100:00080000:0.0:1787829042.696108:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829042.696112:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829042.696117:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829042.696131:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a940 x1874672085741184/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829042.696136:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787829042.696148:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829042.696152:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829042.696154:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829042.696159:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2478 00000100:00080000:0.0:1787829042.696163:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829042.696168:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829042.696173:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:2.0:1787829042.696180:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829042/real 1787829042] req@ffff9e35da718b40 x1874672085740928/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829067 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829042.696187:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7183c0 x1874672085741312/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829042.696202:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787829042.696204:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718b40 x1874672085740928/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829067 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829042.696208:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829042.696213:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829042.696220:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829042.696221:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2478 00000100:00080000:0.0:1787829042.696229:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829042.696234:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:2.0:1787829042.696238:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000008:00100000:0.0:1787829042.696239:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:2.0:1787829042.696248:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085740928:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1787829042.696255:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a1c0 x1874672085741440/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829042.696267:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787829042.696272:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085741056:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787829042.696273:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829042.696278:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829042.696285:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2478 00000100:00080000:0.0:1787829042.696290:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829042.696295:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:2.0:1787829042.696300:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:0.0:1787829042.696301:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829042.696313:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829042/real 1787829042] req@ffff9e35da718f00 x1874672085741056/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829067 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829042.696319:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718780 x1874672085741568/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829042.696333:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787829042.696334:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672085741056/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829067 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829042.696344:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787829042.696348:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829042.696349:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829042.696357:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:0.0:1787829042.696360:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2478 00000100:00100000:2.0:1787829042.696364:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085741056:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787829042.696368:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787829042.696371:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085741184:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787829042.696377:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829042.696383:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829042.696397:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da719a40 x1874672085741696/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829042.696399:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829042.696410:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829042/real 1787829042] req@ffff9e35da71a940 x1874672085741184/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829067 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829042.696413:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829042.696422:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829042.696426:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787829042.696430:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672085741184/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829067 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829042.696433:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2478 00000100:00080000:0.0:1787829042.696439:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829042.696442:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829042.696446:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:2.0:1787829042.696449:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829042.696454:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085741184:192.168.203.141@tcp:38:kworker.0 00000008:00100000:0.0:1787829042.696455:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:2.0:1787829042.696460:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085741312:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829042.696471:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a580 x1874672085741824/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829042.696479:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787829042.696488:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787829042.696491:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829042/real 1787829042] req@ffff9e35da7183c0 x1874672085741312/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829067 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829042.696494:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829042.696496:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829042.696502:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2478 00000100:00080000:0.0:1787829042.696510:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829042.696518:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787829042.696519:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672085741312/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829067 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1787829042.696526:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00080000:2.0:1787829042.696528:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829042.696532:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829042.696535:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085741312:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829042.696544:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085741440:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829042.696546:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718000 x1874672085741952/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829042.696560:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829042.696570:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829042/real 1787829042] req@ffff9e35da71a1c0 x1874672085741440/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829067 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829042.696578:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a1c0 x1874672085741440/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829067 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829042.696584:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829042.696587:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829042.696589:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085741440:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829042.696603:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085741568:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787829042.696615:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829042.696621:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829042/real 1787829042] req@ffff9e35da718780 x1874672085741568/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829067 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829042.696633:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718780 x1874672085741568/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829067 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829042.696641:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829042.696644:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829042.696647:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085741568:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829042.696653:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085741696:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787829042.696672:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829042.696684:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829042/real 1787829042] req@ffff9e35da719a40 x1874672085741696/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829067 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829042.696696:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672085741696/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829067 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829042.696702:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829042.696704:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829042.696706:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085741696:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829042.696711:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085741824:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829042.696726:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829042.696737:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829042/real 1787829042] req@ffff9e35da71a580 x1874672085741824/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829067 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829042.696749:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672085741824/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829067 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829042.696765:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829042.696769:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829042.696772:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085741824:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829042.696778:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085741952:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829042.696793:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829042.696821:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829042/real 1787829042] req@ffff9e35da718000 x1874672085741952/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829067 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829042.696829:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718000 x1874672085741952/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829067 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829042.696836:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829042.696838:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829042.696840:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085741952:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829042.823868:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829042.823939:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829042.823953:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787829042.823983:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829043.847911:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:3.0:1787829043.847943:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829043.847963:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787829043.848050:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829044.871876:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787829044.871924:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829044.871992:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829044.871993:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829045.895975:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829045.895977:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829045.896031:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829045.896034:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829046.919917:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829046.919952:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829046.919973:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787829046.920002:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829047.815900:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829047.815910:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787829047.815914:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829047.815922:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2483 00000100:00080000:0.0:1787829047.815939:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829047.815950:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787829047.815989:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7192c0 x1874672085742080/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829047.816024:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829047.816028:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829047.816030:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829047.816035:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2483 00000100:00080000:0.0:1787829047.816040:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829047.816044:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829047.816056:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829047.816073:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718000 x1874672085742208/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829047.816084:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085742080:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1787829047.816087:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829047.816093:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787829047.816096:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829047.816102:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2483 00000100:00080000:0.0:1787829047.816105:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829047.816109:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829047.816113:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829047.816124:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a580 x1874672085742336/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829047.816131:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829047.816133:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829047.816134:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829047.816137:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2483 00000100:00080000:0.0:1787829047.816140:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829047.816142:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829047.816145:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:0.0:1787829047.816153:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da719a40 x1874672085742464/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829047.816159:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829047.816164:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000800:00000100:2.0:1787829047.816164:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787829047.816168:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829047.816173:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2483 00000100:00080000:0.0:1787829047.816177:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829047.816180:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829047.816184:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:0.0:1787829047.816192:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718780 x1874672085742592/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829047.816198:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829047.816202:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829047.816205:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829047.816210:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2483 00000100:00100000:2.0:1787829047.816214:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829047/real 1787829047] req@ffff9e35da7192c0 x1874672085742080/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829072 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829047.816217:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829047.816226:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829047.816234:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829047.816239:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7192c0 x1874672085742080/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829072 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829047.816249:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1787829047.816251:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a1c0 x1874672085742720/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829047.816255:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829047.816262:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085742080:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1787829047.816272:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829047.816275:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787829047.816277:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829047.816288:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2483 00000100:00100000:2.0:1787829047.816289:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085742208:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787829047.816292:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829047.816298:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829047.816303:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1787829047.816314:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829047.816316:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7183c0 x1874672085742848/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829047.816327:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787829047.816329:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829047/real 1787829047] req@ffff9e35da718000 x1874672085742208/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829072 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829047.816330:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829047.816334:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829047.816342:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2483 00000100:00080000:0.0:1787829047.816347:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787829047.816350:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718000 x1874672085742208/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829072 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829047.816353:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829047.816361:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00080000:2.0:1787829047.816364:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829047.816369:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829047.816373:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a940 x1874672085742976/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829047.816376:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085742208:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829047.816389:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085742336:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787829047.816391:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829047.816395:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829047.816396:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829047.816400:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2483 00000100:00080000:0.0:1787829047.816405:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829047.816407:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829047.816411:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000800:00000100:2.0:1787829047.816415:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829047.816423:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718f00 x1874672085743104/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829047.816441:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829047/real 1787829047] req@ffff9e35da71a580 x1874672085742336/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829072 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829047.816463:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672085742336/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829072 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829047.816470:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829047.816472:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829047.816474:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085742336:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829047.816479:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085742464:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829047.816496:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829047.816503:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829047/real 1787829047] req@ffff9e35da719a40 x1874672085742464/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829072 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829047.816511:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672085742464/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829072 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829047.816517:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829047.816519:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829047.816521:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085742464:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829047.816525:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085742592:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829047.816535:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829047.816540:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829047/real 1787829047] req@ffff9e35da718780 x1874672085742592/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829072 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829047.816548:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718780 x1874672085742592/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829072 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829047.816557:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829047.816559:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829047.816561:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085742592:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829047.816564:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085742720:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787829047.816574:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829047.816579:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829047/real 1787829047] req@ffff9e35da71a1c0 x1874672085742720/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829072 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829047.816586:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a1c0 x1874672085742720/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829072 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829047.816591:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829047.816593:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829047.816595:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085742720:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829047.816611:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085742848:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787829047.816621:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829047.816627:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829047/real 1787829047] req@ffff9e35da7183c0 x1874672085742848/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829072 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829047.816635:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672085742848/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829072 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829047.816639:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829047.816641:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829047.816644:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085742848:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829047.816647:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085742976:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829047.816657:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829047.816663:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829047/real 1787829047] req@ffff9e35da71a940 x1874672085742976/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829072 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829047.816670:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672085742976/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829072 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829047.816675:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829047.816677:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829047.816679:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085742976:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829047.816682:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085743104:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829047.816690:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829047.816696:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829047/real 1787829047] req@ffff9e35da718f00 x1874672085743104/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829072 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829047.816703:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672085743104/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829072 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829047.816708:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829047.816710:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829047.816712:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085743104:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829047.943934:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829047.943940:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829047.943979:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787829047.943993:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829048.967875:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:3.0:1787829048.967894:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829048.967895:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829048.967989:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829049.991958:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829049.991979:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829049.991984:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787829049.992080:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829051.015899:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829051.015906:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787829051.015941:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:0.0:1787829051.016037:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829052.039914:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829052.039914:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787829052.039919:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829052.040010:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829052.935903:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829052.935914:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829052.935918:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829052.935929:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2488 00000100:00080000:1.0:1787829052.935940:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829052.935948:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787829052.935995:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6d00 x1874672085743232/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829052.936020:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829052.936023:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829052.936025:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829052.936028:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2488 00000100:00080000:1.0:1787829052.936030:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829052.936032:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829052.936042:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829052.936052:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4b40 x1874672085743360/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829052.936066:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829052.936069:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829052.936070:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829052.936075:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2488 00000100:00080000:1.0:1787829052.936078:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787829052.936079:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085743232:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787829052.936082:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829052.936089:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829052.936096:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4000 x1874672085743488/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829052.936102:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829052.936105:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829052.936106:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829052.936109:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2488 00000100:00080000:1.0:1787829052.936111:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829052.936114:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829052.936123:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:1.0:1787829052.936132:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5e00 x1874672085743616/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829052.936138:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829052.936140:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829052.936141:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829052.936144:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2488 00000100:00080000:1.0:1787829052.936151:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829052.936153:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829052.936156:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:1.0:1787829052.936164:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7480 x1874672085743744/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829052.936171:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829052.936174:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829052.936175:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829052.936178:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2488 00000100:00080000:1.0:1787829052.936179:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829052.936183:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829052.936186:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829052.936198:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4780 x1874672085743872/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829052.936204:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829052.936206:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829052.936207:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829052.936210:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2488 00000100:00080000:1.0:1787829052.936213:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787829052.936216:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085743360:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787829052.936218:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829052.936224:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829052.936235:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5a40 x1874672085744000/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829052.936247:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829052.936250:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829052.936251:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829052.936254:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2488 00000100:00080000:1.0:1787829052.936257:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787829052.936259:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085743488:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787829052.936262:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829052.936269:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:1.0:1787829052.936281:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7840 x1874672085744128/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829052.936288:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829052.936291:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829052.936292:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829052.936295:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2488 00000100:00080000:1.0:1787829052.936298:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829052.936303:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787829052.936304:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085743616:192.168.203.141@tcp:8:kworker.0 00000008:00100000:1.0:1787829052.936306:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:1.0:1787829052.936317:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f70c0 x1874672085744256/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829052.936344:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085743744:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829052.936374:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085743872:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829052.936398:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085744000:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829052.936416:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085744128:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829052.936435:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085744256:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829053.063905:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829053.063905:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787829053.063941:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787829053.064049:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829054.087954:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829054.087954:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787829054.087966:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:0.0:1787829054.088078:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829055.111918:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829055.111927:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787829055.111982:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829055.112060:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1787829056.008108:0:2397:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.141:988 00000400:00000100:0.0:1787829056.008152:0:2397:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.141@tcp at host 192.168.203.141 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1787829056.008169:0:2397:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:0.0:1787829056.008225:0:2397:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000800:00000100:0.0:1787829056.008237:0:2397:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:0.0:1787829056.008246:0:2397:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:0.0:1787829056.008255:0:2397:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:0.0:1787829056.008263:0:2397:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:0.0:1787829056.008272:0:2397:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:0.0:1787829056.008280:0:2397:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:0.0:1787829056.008289:0:2397:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:2.0:1787829056.008383:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829052/real 1787829056] req@ffff9e35da6f5a40 x1874672085744000/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829077 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829056.008410:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5a40 x1874672085744000/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829077 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829056.008430:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829056.008436:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829056.008441:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085744000:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829056.008448:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829052/real 1787829056] req@ffff9e35da6f7840 x1874672085744128/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829077 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829056.008460:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7840 x1874672085744128/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829077 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829056.008469:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829056.008473:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829056.008476:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085744128:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829056.008483:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829052/real 1787829056] req@ffff9e35da6f70c0 x1874672085744256/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829077 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829056.008493:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f70c0 x1874672085744256/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829077 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829056.008501:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829056.008505:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829056.008508:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085744256:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829056.008515:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829052/real 1787829056] req@ffff9e35da6f4b40 x1874672085743360/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829077 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829056.008527:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4b40 x1874672085743360/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829077 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829056.008537:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829056.008541:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829056.008544:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085743360:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829056.008552:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829052/real 1787829056] req@ffff9e35da6f4000 x1874672085743488/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829077 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829056.008564:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4000 x1874672085743488/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829077 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829056.008572:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829056.008576:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829056.008579:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085743488:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829056.008586:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829052/real 1787829056] req@ffff9e35da6f5e00 x1874672085743616/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829077 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829056.008598:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5e00 x1874672085743616/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829077 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829056.008607:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829056.008610:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829056.008614:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085743616:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829056.008621:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829052/real 1787829056] req@ffff9e35da6f7480 x1874672085743744/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829077 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829056.008639:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7480 x1874672085743744/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829077 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829056.008649:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829056.008653:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829056.008657:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085743744:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829056.008665:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829052/real 1787829056] req@ffff9e35da6f4780 x1874672085743872/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829077 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829056.008678:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4780 x1874672085743872/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829077 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829056.008687:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829056.008692:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829056.008695:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085743872:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829056.008702:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829052/real 1787829056] req@ffff9e35da6f6d00 x1874672085743232/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829077 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829056.008716:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6d00 x1874672085743232/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829077 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829056.008725:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829056.008729:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829056.008732:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085743232:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1787829056.135911:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787829056.135937:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829056.135978:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787829056.136045:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829057.159968:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829057.159969:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829057.160009:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:0.0:1787829057.160023:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829058.055937:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829058.055948:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829058.055953:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829058.055965:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2493 00000100:00080000:1.0:1787829058.055979:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829058.055988:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787829058.056038:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6940 x1874672085744384/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829058.056084:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829058.056088:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829058.056090:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829058.056095:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2493 00000100:00080000:1.0:1787829058.056100:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829058.056106:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829058.056119:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829058.056137:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6580 x1874672085744512/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829058.056147:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085744384:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787829058.056153:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829058.056163:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829058.056165:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829058.056170:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2493 00000100:00080000:1.0:1787829058.056175:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829058.056179:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829058.056185:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829058.056200:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f43c0 x1874672085744640/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829058.056211:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829058.056215:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829058.056217:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829058.056221:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2493 00000100:00080000:1.0:1787829058.056236:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829058.056241:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829058.056247:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000800:00000100:2.0:1787829058.056249:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829058.056264:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f61c0 x1874672085744768/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829058.056274:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829058.056278:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829058.056280:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829058.056285:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2493 00000100:00080000:1.0:1787829058.056293:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829058.056298:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829058.056304:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:1.0:1787829058.056316:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4f00 x1874672085744896/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829058.056332:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829058/real 1787829058] req@ffff9e35da6f6940 x1874672085744384/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829083 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829058.056348:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829058.056354:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829058.056356:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829058.056362:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2493 00000100:00080000:1.0:1787829058.056368:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787829058.056372:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6940 x1874672085744384/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829083 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829058.056376:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829058.056388:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829058.056403:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6d00 x1874672085745024/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829058.056408:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829058.056419:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829058.056420:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:1.0:1787829058.056427:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829058.056431:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787829058.056431:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085744384:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787829058.056441:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2493 00000100:00080000:1.0:1787829058.056447:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829058.056452:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829058.056459:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829058.056473:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4780 x1874672085745152/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829058.056479:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085744512:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787829058.056491:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829058.056496:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829058.056498:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829058.056504:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2493 00000100:00080000:1.0:1787829058.056509:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829058.056515:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:2.0:1787829058.056522:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:1.0:1787829058.056525:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:1.0:1787829058.056541:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7480 x1874672085745280/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829058.056543:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829058/real 1787829058] req@ffff9e35da6f6580 x1874672085744512/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829083 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829058.056582:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787829058.056583:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6580 x1874672085744512/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829083 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829058.056590:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829058.056594:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829058.056603:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829058.056604:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2493 00000100:00080000:1.0:1787829058.056613:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829058.056614:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:1.0:1787829058.056620:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787829058.056622:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085744512:192.168.203.141@tcp:38:kworker.0 00000008:00100000:1.0:1787829058.056628:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:2.0:1787829058.056635:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085744640:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829058.056647:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5e00 x1874672085745408/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829058.056668:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829058.056684:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829058/real 1787829058] req@ffff9e35da6f43c0 x1874672085744640/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829083 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829058.056708:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f43c0 x1874672085744640/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829083 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829058.056717:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829058.056721:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829058.056725:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085744640:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829058.056732:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085744768:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829058.056759:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829058.056773:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829058/real 1787829058] req@ffff9e35da6f61c0 x1874672085744768/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829083 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829058.056785:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f61c0 x1874672085744768/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829083 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829058.056793:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829058.056854:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829058.056860:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085744768:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829058.056867:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085744896:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829058.056890:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829058.056903:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829058/real 1787829058] req@ffff9e35da6f4f00 x1874672085744896/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829083 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829058.056918:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4f00 x1874672085744896/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829083 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829058.056934:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829058.056939:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829058.056942:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085744896:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829058.056950:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085745024:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787829058.056975:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829058.056988:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829058/real 1787829058] req@ffff9e35da6f6d00 x1874672085745024/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829083 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829058.057004:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6d00 x1874672085745024/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829083 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829058.057014:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829058.057017:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829058.057020:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085745024:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829058.057048:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085745152:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787829058.057070:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829058.057081:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829058/real 1787829058] req@ffff9e35da6f4780 x1874672085745152/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829083 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829058.057095:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4780 x1874672085745152/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829083 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829058.057104:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829058.057107:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829058.057110:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085745152:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829058.057117:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085745280:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829058.057137:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829058.057148:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829058/real 1787829058] req@ffff9e35da6f7480 x1874672085745280/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829083 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829058.057159:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7480 x1874672085745280/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829083 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829058.057168:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829058.057171:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829058.057175:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085745280:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829058.057181:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085745408:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829058.057199:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829058.057210:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829058/real 1787829058] req@ffff9e35da6f5e00 x1874672085745408/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829083 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829058.057222:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5e00 x1874672085745408/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829083 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829058.057230:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829058.057233:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829058.057236:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085745408:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829058.183953:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829058.183958:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829058.183962:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829058.184035:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829059.207941:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829059.207941:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829059.207950:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787829059.207989:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829060.231938:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829060.231950:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829060.231982:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787829060.231984:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829061.255963:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829061.255963:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829061.256008:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829061.256018:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829062.279910:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829062.279910:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829062.279977:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:0.0:1787829062.280036:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829063.175925:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829063.175936:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829063.175940:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829063.175951:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2498 00000100:00080000:2.0:1787829063.175963:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829063.175970:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787829063.176015:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c3c0 x1874672085745536/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829063.176044:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829063.176048:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829063.176049:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829063.176052:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2498 00000100:00080000:2.0:1787829063.176055:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829063.176058:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829063.176067:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829063.176076:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498ed00 x1874672085745664/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829063.176085:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829063.176088:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829063.176091:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829063.176096:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2498 00000100:00080000:2.0:1787829063.176098:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829063.176101:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829063.176104:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829063.176110:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498de00 x1874672085745792/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829063.176118:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829063.176122:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829063.176123:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829063.176126:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2498 00000100:00080000:2.0:1787829063.176128:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829063.176135:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787829063.176139:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085745536:192.168.203.141@tcp:250:kworker.0 00000008:00100000:2.0:1787829063.176140:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:2.0:1787829063.176149:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498fc00 x1874672085745920/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829063.176159:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829063.176162:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829063.176163:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829063.176168:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2498 00000100:00080000:2.0:1787829063.176170:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829063.176173:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829063.176176:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:2.0:1787829063.176185:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e940 x1874672085746048/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829063.176192:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829063.176196:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829063.176197:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829063.176201:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2498 00000100:00080000:2.0:1787829063.176205:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829063.176208:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829063.176211:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829063.176218:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c780 x1874672085746176/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829063.176225:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1787829063.176230:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1787829063.176231:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829063.176242:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829063.176249:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2498 00000100:00080000:2.0:1787829063.176254:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829063.176257:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829063.176261:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829063.176274:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498d680 x1874672085746304/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829063.176280:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829063.176282:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829063.176284:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829063.176288:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2498 00000100:00080000:2.0:1787829063.176292:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829063.176296:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829063.176301:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:0.0:1787829063.176305:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829063/real 1787829063] req@ffff9e35c498c3c0 x1874672085745536/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829088 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829063.176313:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cf00 x1874672085746432/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829063.176336:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829063.176339:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829063.176341:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787829063.176345:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c3c0 x1874672085745536/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829088 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829063.176351:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2498 00000100:00080000:2.0:1787829063.176366:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829063.176370:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829063.176378:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:0.0:1787829063.176379:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829063.176385:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085745536:192.168.203.141@tcp:250:kworker.0 00000008:00100000:2.0:1787829063.176392:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:0.0:1787829063.176396:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085745664:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829063.176415:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c000 x1874672085746560/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829063.176435:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829063.176458:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829063/real 1787829063] req@ffff9e35c498ed00 x1874672085745664/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829088 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829063.176471:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498ed00 x1874672085745664/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829088 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829063.176478:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829063.176481:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829063.176484:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085745664:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829063.176511:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085745792:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829063.176535:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829063.176548:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829063/real 1787829063] req@ffff9e35c498de00 x1874672085745792/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829088 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829063.176561:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498de00 x1874672085745792/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829088 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829063.176569:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829063.176572:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829063.176585:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085745792:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829063.176592:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085745920:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829063.176613:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829063.176625:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829063/real 1787829063] req@ffff9e35c498fc00 x1874672085745920/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829088 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829063.176633:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498fc00 x1874672085745920/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829088 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829063.176639:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829063.176641:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829063.176644:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085745920:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829063.176648:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085746048:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829063.176662:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829063.176672:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829063/real 1787829063] req@ffff9e35c498e940 x1874672085746048/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829088 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829063.176684:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e940 x1874672085746048/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829088 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829063.176694:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829063.176697:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829063.176701:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085746048:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829063.176708:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085746176:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829063.176727:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829063.176736:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829063/real 1787829063] req@ffff9e35c498c780 x1874672085746176/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829088 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829063.176744:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c780 x1874672085746176/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829088 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829063.176750:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829063.176752:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829063.176754:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085746176:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829063.176759:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085746304:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829063.176770:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829063.176776:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829063/real 1787829063] req@ffff9e35c498d680 x1874672085746304/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829088 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829063.176845:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498d680 x1874672085746304/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829088 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829063.176855:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829063.176858:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829063.176862:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085746304:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829063.176869:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085746432:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829063.176893:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829063.176905:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829063/real 1787829063] req@ffff9e35c498cf00 x1874672085746432/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829088 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829063.176917:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498cf00 x1874672085746432/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829088 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829063.176924:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829063.176926:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829063.176928:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085746432:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829063.176933:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085746560:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829063.176949:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829063.176956:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829063/real 1787829063] req@ffff9e35c498c000 x1874672085746560/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829088 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829063.176963:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c000 x1874672085746560/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829088 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829063.176969:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829063.176971:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829063.176974:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085746560:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829063.303919:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829063.303921:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829063.303966:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:1.0:1787829063.303982:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829064.327902:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829064.327964:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829064.327976:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829064.327985:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829065.351931:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829065.351964:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829065.351997:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:0.0:1787829065.352013:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829066.375950:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829066.375971:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829066.375994:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829066.376015:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829067.399949:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829067.399950:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829067.399997:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787829067.400012:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829068.295986:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829068.296008:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829068.296014:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829068.296031:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2503 00000100:00080000:2.0:1787829068.296049:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829068.296060:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787829068.296112:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cb40 x1874672085746688/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829068.296152:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829068.296157:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829068.296160:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829068.296167:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2503 00000100:00080000:2.0:1787829068.296175:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829068.296182:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829068.296197:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829068.296219:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e1c0 x1874672085746816/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829068.296228:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085746688:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1787829068.296245:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829068.296253:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829068.296257:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829068.296264:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2503 00000100:00080000:2.0:1787829068.296271:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829068.296276:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829068.296283:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829068.296303:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f480 x1874672085746944/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829068.296306:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1787829068.296324:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829068.296329:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829068.296332:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829068.296340:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2503 00000100:00080000:2.0:1787829068.296347:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829068.296354:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829068.296363:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:0.0:1787829068.296364:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829068/real 1787829068] req@ffff9e35c498cb40 x1874672085746688/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829093 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829068.296402:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f0c0 x1874672085747072/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829068.296403:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498cb40 x1874672085746688/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829093 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829068.296457:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829068.296462:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829068.296465:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:2.0:1787829068.296475:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:0.0:1787829068.296480:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085746688:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1787829068.296481:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829068.296514:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2503 00000100:00080000:2.0:1787829068.296521:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829068.296527:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829068.296534:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:0.0:1787829068.296551:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085746816:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829068.296560:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e580 x1874672085747200/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829068.296598:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829068.296603:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829068.296606:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829068.296615:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2503 00000800:00000100:0.0:1787829068.296624:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1787829068.296635:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829068.296666:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787829068.296668:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829068/real 1787829068] req@ffff9e35c498e1c0 x1874672085746816/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829093 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1787829068.296686:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829068.296697:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e1c0 x1874672085746816/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829093 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829068.296722:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:2.0:1787829068.296730:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f840 x1874672085747328/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829068.296732:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829068.296769:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085746816:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829068.296862:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085746944:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1787829068.296880:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829068.296887:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829068.296892:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829068.296903:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2503 00000800:00000100:0.0:1787829068.296909:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1787829068.296917:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829068.296927:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829068.296937:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829068.296938:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829068/real 1787829068] req@ffff9e35c498f480 x1874672085746944/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829093 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829068.296962:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f480 x1874672085746944/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829093 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829068.296969:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cb40 x1874672085747456/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829068.296979:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829068.296990:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829068.297006:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085746944:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829068.297023:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085747072:192.168.203.141@tcp:8:kworker.0 00000100:00080000:2.0:1787829068.297024:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829068.297044:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829068.297049:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829068.297058:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2503 00000800:00000100:0.0:1787829068.297064:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1787829068.297072:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787829068.297081:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829068/real 1787829068] req@ffff9e35c498f0c0 x1874672085747072/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829093 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829068.297085:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829068.297108:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:0.0:1787829068.297110:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f0c0 x1874672085747072/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829093 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829068.297129:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829068.297133:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829068.297144:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085747072:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829068.297145:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c000 x1874672085747584/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829068.297166:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829068.297172:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:0.0:1787829068.297176:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085747200:192.168.203.141@tcp:8:kworker.0 00000100:00080000:2.0:1787829068.297178:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829068.297192:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2503 00000100:00080000:2.0:1787829068.297199:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1787829068.297203:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1787829068.297212:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787829068.297220:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829068/real 1787829068] req@ffff9e35c498e580 x1874672085747200/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829093 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1787829068.297225:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:0.0:1787829068.297248:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e580 x1874672085747200/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829093 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829068.297257:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cf00 x1874672085747712/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829068.297277:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829068.297292:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829068.297300:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085747200:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829068.297308:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085747328:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829068.297329:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829068.297360:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829068/real 1787829068] req@ffff9e35c498f840 x1874672085747328/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829093 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829068.297379:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f840 x1874672085747328/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829093 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829068.297392:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829068.297397:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829068.297402:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085747328:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829068.297408:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085747456:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829068.297429:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829068.297447:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829068/real 1787829068] req@ffff9e35c498cb40 x1874672085747456/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829093 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829068.297457:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498cb40 x1874672085747456/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829093 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829068.297475:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829068.297479:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829068.297484:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085747456:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829068.297492:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085747584:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829068.297531:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829068.297546:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829068/real 1787829068] req@ffff9e35c498c000 x1874672085747584/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829093 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829068.297559:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c000 x1874672085747584/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829093 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829068.297579:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829068.297598:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829068.297601:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085747584:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829068.297623:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085747712:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829068.297645:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829068.297682:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829068/real 1787829068] req@ffff9e35c498cf00 x1874672085747712/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829093 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829068.297697:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498cf00 x1874672085747712/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829093 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829068.297707:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829068.297711:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829068.297716:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085747712:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829068.423947:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829068.423947:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829068.424002:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829068.424011:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829069.447912:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829069.447944:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829069.447945:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829069.447953:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829070.471936:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829070.471978:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829070.471999:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:3.0:1787829070.472017:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829071.495933:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:3.0:1787829071.495933:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829071.495970:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829071.496054:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829072.519874:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:3.0:1787829072.519879:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829072.519886:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829072.519956:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829073.416007:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829073.416023:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829073.416030:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829073.416072:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2508 00000100:00080000:1.0:1787829073.416089:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829073.416099:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787829073.416171:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4000 x1874672085747840/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829073.416203:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829073.416208:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829073.416210:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829073.416215:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2508 00000100:00080000:1.0:1787829073.416219:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829073.416225:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829073.416237:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829073.416253:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085747840:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1787829073.416263:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4b40 x1874672085747968/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829073.416277:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829073.416281:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829073.416283:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829073.416287:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2508 00000100:00080000:1.0:1787829073.416291:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829073.416295:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829073.416302:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829073.416317:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f70c0 x1874672085748096/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829073.416327:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1787829073.416334:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787829073.416334:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829073.416339:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829073.416346:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2508 00000100:00080000:1.0:1787829073.416350:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829073.416356:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829073.416361:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:1.0:1787829073.416376:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7840 x1874672085748224/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829073.416387:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787829073.416390:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829073/real 1787829073] req@ffff9e35da6f4000 x1874672085747840/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829098 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829073.416392:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829073.416396:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829073.416416:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2508 00000100:00080000:1.0:1787829073.416421:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787829073.416424:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4000 x1874672085747840/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829098 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829073.416430:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:0.0:1787829073.416439:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:1.0:1787829073.416440:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00080000:0.0:1787829073.416447:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829073.416451:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085747840:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1787829073.416458:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5a40 x1874672085748352/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829073.416469:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829073.416473:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:0.0:1787829073.416477:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085747968:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787829073.416478:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829073.416489:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2508 00000100:00080000:1.0:1787829073.416493:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829073.416510:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:0.0:1787829073.416516:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:1.0:1787829073.416519:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829073.416534:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829073/real 1787829073] req@ffff9e35da6f4b40 x1874672085747968/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829098 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829073.416538:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4000 x1874672085748480/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829073.416567:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787829073.416569:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4b40 x1874672085747968/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829098 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829073.416575:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829073.416580:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829073.416583:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829073.416588:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2508 00000100:00080000:0.0:1787829073.416589:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829073.416597:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085747968:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787829073.416598:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787829073.416605:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085748096:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787829073.416607:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829073.416616:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1787829073.416639:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829073.416644:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5e00 x1874672085748608/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829073.416650:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829073/real 1787829073] req@ffff9e35da6f70c0 x1874672085748096/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829098 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829073.416666:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787829073.416669:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f70c0 x1874672085748096/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829098 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829073.416673:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829073.416679:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829073.416684:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829073.416689:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:1.0:1787829073.416689:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2508 00000100:00100000:0.0:1787829073.416696:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085748096:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787829073.416698:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787829073.416702:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085748224:192.168.203.141@tcp:8:kworker.0 00000100:00080000:1.0:1787829073.416708:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829073.416731:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000800:00000100:0.0:1787829073.416740:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829073.416752:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829073/real 1787829073] req@ffff9e35da6f7840 x1874672085748224/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829098 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829073.416760:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7480 x1874672085748736/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829073.416766:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7840 x1874672085748224/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829098 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829073.416788:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829073.416789:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829073.416793:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:1.0:1787829073.416822:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829073.416826:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829073.416832:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2508 00000100:00080000:1.0:1787829073.416837:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829073.416842:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787829073.416849:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085748224:192.168.203.141@tcp:8:kworker.0 00000008:00100000:1.0:1787829073.416851:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:0.0:1787829073.416858:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085748352:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829073.416869:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4780 x1874672085748864/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829073.416878:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829073.416892:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829073/real 1787829073] req@ffff9e35da6f5a40 x1874672085748352/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829098 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829073.416914:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5a40 x1874672085748352/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829098 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829073.416921:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829073.416924:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829073.416926:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085748352:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829073.416945:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085748480:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829073.416957:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829073.416964:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829073/real 1787829073] req@ffff9e35da6f4000 x1874672085748480/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829098 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829073.416983:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4000 x1874672085748480/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829098 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829073.416989:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829073.416990:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829073.416992:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085748480:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829073.416996:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085748608:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829073.417006:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829073.417012:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829073/real 1787829073] req@ffff9e35da6f5e00 x1874672085748608/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829098 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829073.417022:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5e00 x1874672085748608/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829098 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829073.417038:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829073.417042:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829073.417046:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085748608:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829073.417053:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085748736:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829073.417084:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829073.417092:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829073/real 1787829073] req@ffff9e35da6f7480 x1874672085748736/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829098 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829073.417100:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7480 x1874672085748736/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829098 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829073.417107:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829073.417109:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829073.417112:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085748736:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829073.417116:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085748864:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829073.417129:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829073.417146:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829073/real 1787829073] req@ffff9e35da6f4780 x1874672085748864/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829098 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829073.417162:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4780 x1874672085748864/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829098 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829073.417168:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829073.417171:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829073.417174:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085748864:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829073.543878:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829073.543878:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:3.0:1787829073.543883:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829073.543956:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829074.567924:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829074.567924:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:3.0:1787829074.567934:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829074.568045:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829075.591925:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829075.591926:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829075.591953:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829075.591964:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829076.615923:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829076.615956:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829076.615963:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829076.615969:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829077.639928:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829077.639928:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829077.639957:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787829077.639975:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829078.535916:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829078.535926:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829078.535931:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829078.535942:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2513 00000100:00080000:1.0:1787829078.535953:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829078.535960:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787829078.536004:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6d00 x1874672085748992/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829078.536034:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829078.536039:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829078.536040:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829078.536046:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2513 00000100:00080000:1.0:1787829078.536049:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829078.536053:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829078.536066:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829078.536081:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4f00 x1874672085749120/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829078.536090:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085748992:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787829078.536095:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829078.536099:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829078.536101:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829078.536104:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2513 00000100:00080000:1.0:1787829078.536107:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829078.536110:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829078.536115:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829078.536124:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f61c0 x1874672085749248/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829078.536131:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829078.536134:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829078.536135:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829078.536138:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2513 00000100:00080000:1.0:1787829078.536154:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829078.536157:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829078.536160:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000800:00000100:0.0:1787829078.536164:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829078.536172:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f43c0 x1874672085749376/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829078.536181:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829078.536183:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829078.536184:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829078.536188:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2513 00000100:00080000:1.0:1787829078.536190:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829078.536193:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829078.536197:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:1.0:1787829078.536210:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6580 x1874672085749504/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829078.536213:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829078/real 1787829078] req@ffff9e35da6f6d00 x1874672085748992/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829103 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829078.536239:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787829078.536244:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6d00 x1874672085748992/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829103 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829078.536245:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829078.536251:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829078.536261:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2513 00000100:00080000:0.0:1787829078.536264:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829078.536265:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829078.536273:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:0.0:1787829078.536274:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829078.536280:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085748992:192.168.203.141@tcp:250:kworker.0 00000008:00100000:1.0:1787829078.536280:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829078.536299:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6940 x1874672085749632/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829078.536306:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085749120:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787829078.536311:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829078.536315:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829078.536316:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829078.536320:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2513 00000100:00080000:1.0:1787829078.536326:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1787829078.536330:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787829078.536332:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829078.536338:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829078.536345:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829078/real 1787829078] req@ffff9e35da6f4f00 x1874672085749120/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829103 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829078.536349:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6d00 x1874672085749760/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829078.536364:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4f00 x1874672085749120/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829103 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829078.536364:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829078.536372:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829078.536376:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829078.536385:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829078.536385:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2513 00000100:00080000:1.0:1787829078.536388:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829078.536392:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:1.0:1787829078.536396:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787829078.536400:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085749120:192.168.203.141@tcp:38:kworker.0 00000008:00100000:1.0:1787829078.536402:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:0.0:1787829078.536409:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085749248:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829078.536414:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4780 x1874672085749888/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829078.536422:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829078.536427:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000800:00000100:0.0:1787829078.536428:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787829078.536433:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829078.536438:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2513 00000100:00100000:0.0:1787829078.536441:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829078/real 1787829078] req@ffff9e35da6f61c0 x1874672085749248/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829103 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829078.536442:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829078.536449:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829078.536457:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:0.0:1787829078.536462:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f61c0 x1874672085749248/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829103 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829078.536468:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7480 x1874672085750016/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829078.536473:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829078.536481:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829078.536487:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085749248:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829078.536498:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085749376:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829078.536515:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829078.536524:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829078/real 1787829078] req@ffff9e35da6f43c0 x1874672085749376/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829103 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829078.536532:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f43c0 x1874672085749376/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829103 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829078.536537:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829078.536539:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829078.536541:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085749376:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829078.536545:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085749504:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829078.536555:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829078.536562:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829078/real 1787829078] req@ffff9e35da6f6580 x1874672085749504/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829103 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829078.536569:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6580 x1874672085749504/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829103 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829078.536574:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829078.536576:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829078.536578:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085749504:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829078.536592:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085749632:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829078.536602:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829078.536608:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829078/real 1787829078] req@ffff9e35da6f6940 x1874672085749632/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829103 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829078.536616:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6940 x1874672085749632/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829103 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829078.536622:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829078.536623:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829078.536625:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085749632:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829078.536629:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085749760:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829078.536638:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829078.536644:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829078/real 1787829078] req@ffff9e35da6f6d00 x1874672085749760/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829103 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829078.536651:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6d00 x1874672085749760/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829103 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829078.536657:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829078.536659:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829078.536661:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085749760:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829078.536665:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085749888:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829078.536691:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829078.536698:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829078/real 1787829078] req@ffff9e35da6f4780 x1874672085749888/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829103 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829078.536705:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4780 x1874672085749888/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829103 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829078.536710:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829078.536712:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829078.536714:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085749888:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829078.536717:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085750016:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829078.536728:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829078.536733:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829078/real 1787829078] req@ffff9e35da6f7480 x1874672085750016/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829103 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829078.536740:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7480 x1874672085750016/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829103 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829078.536745:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829078.536747:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829078.536750:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085750016:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829078.663964:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829078.663968:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787829078.664031:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829078.664181:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829079.687930:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829079.687951:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:3.0:1787829079.687970:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829079.688091:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829080.711957:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829080.711965:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:3.0:1787829080.711965:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787829080.712130:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829081.735934:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829081.735968:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829081.736144:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787829081.736233:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829082.759975:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829082.759979:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829082.759979:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829082.760056:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829083.655942:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829083.655953:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829083.655957:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829083.655968:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2519 00000100:00080000:1.0:1787829083.655980:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829083.655987:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787829083.656037:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5e00 x1874672085750144/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829083.656070:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829083.656073:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829083.656074:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829083.656078:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2519 00000100:00080000:1.0:1787829083.656081:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829083.656084:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829083.656094:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829083.656108:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4000 x1874672085750272/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829083.656120:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829083.656124:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829083.656127:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787829083.656137:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085750144:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787829083.656137:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2519 00000100:00080000:1.0:1787829083.656147:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829083.656155:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829083.656162:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829083.656179:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5a40 x1874672085750400/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829083.656189:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829083.656193:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829083.656194:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829083.656199:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2519 00000100:00080000:1.0:1787829083.656212:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1787829083.656219:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787829083.656221:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829083.656230:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:1.0:1787829083.656249:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7840 x1874672085750528/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829083.656260:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787829083.656262:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829083/real 1787829083] req@ffff9e35da6f5e00 x1874672085750144/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829108 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829083.656271:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829083.656278:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829083.656289:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2519 00000100:00100000:0.0:1787829083.656297:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5e00 x1874672085750144/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829108 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829083.656298:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829083.656312:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829083.656323:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00080000:0.0:1787829083.656329:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829083.656338:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829083.656343:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f70c0 x1874672085750656/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829083.656346:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085750144:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787829083.656371:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829083.656376:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829083.656380:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787829083.656385:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085750272:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787829083.656391:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2519 00000100:00080000:1.0:1787829083.656400:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829083.656408:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:0.0:1787829083.656414:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:1.0:1787829083.656415:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829083.656427:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829083/real 1787829083] req@ffff9e35da6f4000 x1874672085750272/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829108 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829083.656434:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4b40 x1874672085750784/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829083.656455:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4000 x1874672085750272/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829108 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829083.656475:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829083.656475:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829083.656487:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:1.0:1787829083.656491:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829083.656497:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787829083.656498:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085750272:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787829083.656514:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2519 00000100:00100000:0.0:1787829083.656516:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085750400:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787829083.656525:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829083.656532:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:0.0:1787829083.656537:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:1.0:1787829083.656539:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829083.656549:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829083/real 1787829083] req@ffff9e35da6f5a40 x1874672085750400/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829108 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829083.656555:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5e00 x1874672085750912/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829083.656577:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5a40 x1874672085750400/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829108 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829083.656588:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829083.656601:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829083.656607:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829083.656613:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829083.656617:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2519 00000100:00080000:0.0:1787829083.656624:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:1.0:1787829083.656626:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787829083.656630:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085750400:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787829083.656633:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829083.656660:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:0.0:1787829083.656662:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085750528:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829083.656679:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7480 x1874672085751040/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829083.656688:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1787829083.656694:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787829083.656698:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829083.656702:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829083.656708:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2519 00000100:00100000:0.0:1787829083.656713:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829083/real 1787829083] req@ffff9e35da6f7840 x1874672085750528/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829108 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829083.656714:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829083.656727:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829083.656742:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:0.0:1787829083.656747:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7840 x1874672085750528/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829108 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829083.656757:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1787829083.656758:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4780 x1874672085751168/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829083.656769:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829083.656783:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085750528:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829083.656791:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085750656:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829083.656845:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829083.656856:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829083/real 1787829083] req@ffff9e35da6f70c0 x1874672085750656/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829108 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829083.656869:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f70c0 x1874672085750656/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829108 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829083.656875:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829083.656877:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829083.656879:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085750656:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829083.656896:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085750784:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829083.656908:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829083.656915:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829083/real 1787829083] req@ffff9e35da6f4b40 x1874672085750784/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829108 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829083.656922:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4b40 x1874672085750784/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829108 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829083.656928:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829083.656930:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829083.656932:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085750784:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829083.656936:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085750912:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829083.656945:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829083.656951:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829083/real 1787829083] req@ffff9e35da6f5e00 x1874672085750912/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829108 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829083.656958:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5e00 x1874672085750912/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829108 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829083.656968:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829083.656970:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829083.656972:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085750912:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829083.656976:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085751040:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829083.656993:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829083.657003:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829083/real 1787829083] req@ffff9e35da6f7480 x1874672085751040/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829108 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829083.657016:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7480 x1874672085751040/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829108 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829083.657027:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829083.657030:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829083.657034:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085751040:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829083.657041:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085751168:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829083.657060:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829083.657072:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829083/real 1787829083] req@ffff9e35da6f4780 x1874672085751168/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829108 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829083.657085:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4780 x1874672085751168/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829108 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829083.657094:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829083.657097:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829083.657100:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085751168:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829083.783888:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829083.783898:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829083.783938:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:0.0:1787829083.783981:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829084.807945:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829084.807958:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829084.807981:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787829084.808058:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829085.831883:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829085.831887:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787829085.831932:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829085.831955:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829086.855914:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829086.855960:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787829086.855961:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829086.855968:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829087.879863:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829087.879901:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829087.879919:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829087.879934:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829088.776020:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829088.776036:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787829088.776042:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829088.776056:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2524 00000100:00080000:3.0:1787829088.776072:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829088.776083:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787829088.776150:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499a40 x1874672085751296/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829088.776214:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829088.776223:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829088.776228:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829088.776238:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2524 00000100:00080000:3.0:1787829088.776247:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787829088.776258:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085751296:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1787829088.776261:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829088.776293:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829088.776350:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498780 x1874672085751424/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829088.776366:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1787829088.776395:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829088.776403:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787829088.776408:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829088.776420:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2524 00000100:00100000:0.0:1787829088.776423:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829088/real 1787829088] req@ffff9e35c6499a40 x1874672085751296/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829113 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829088.776436:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829088.776456:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829088.776472:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829088.776473:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499a40 x1874672085751296/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829113 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829088.776499:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829088.776504:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829088.776508:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085751296:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1787829088.776516:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a940 x1874672085751552/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829088.776544:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085751424:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1787829088.776545:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829088.776562:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787829088.776568:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829088.776581:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2524 00000800:00000100:0.0:1787829088.776582:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1787829088.776596:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787829088.776618:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829088/real 1787829088] req@ffff9e35c6498780 x1874672085751424/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829113 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829088.776636:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498780 x1874672085751424/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829113 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829088.776654:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829088.776669:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00080000:0.0:1787829088.776672:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829088.776684:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829088.776689:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085751424:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787829088.776707:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498b40 x1874672085751680/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829088.776714:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085751552:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1787829088.776753:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829088.776763:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787829088.776768:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1787829088.776777:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1787829088.776784:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2524 00000100:00100000:0.0:1787829088.776799:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829088/real 1787829088] req@ffff9e35c649a940 x1874672085751552/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829113 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829088.776886:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a940 x1874672085751552/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829113 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829088.776889:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829088.776895:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829088.776907:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:3.0:1787829088.776909:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787829088.776916:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085751552:192.168.203.141@tcp:38:kworker.0 00000008:00100000:3.0:1787829088.776937:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:0.0:1787829088.776980:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085751680:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787829088.776981:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499680 x1874672085751808/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829088.777019:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829088.777025:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829088.777029:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1787829088.777038:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1787829088.777045:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2524 00000100:00080000:3.0:1787829088.777057:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787829088.777062:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829088/real 1787829088] req@ffff9e35c6498b40 x1874672085751680/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829113 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829088.777069:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829088.777090:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829088.777119:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498b40 x1874672085751680/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829113 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829088.777122:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498f00 x1874672085751936/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829088.777186:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829088.777197:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829088.777207:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085751680:192.168.203.141@tcp:8:kworker.0 00000100:00080000:3.0:1787829088.777252:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787829088.777260:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085751808:192.168.203.141@tcp:8:kworker.0 00000100:00080000:3.0:1787829088.777265:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787829088.777271:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829088.777281:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2524 00000100:00080000:3.0:1787829088.777288:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1787829088.777298:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1787829088.777300:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829088.777312:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829088.777333:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b840 x1874672085752064/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829088.777357:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829088/real 1787829088] req@ffff9e35c6499680 x1874672085751808/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829113 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829088.777364:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829088.777379:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787829088.777388:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829088.777399:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2524 00000100:00080000:3.0:1787829088.777404:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787829088.777412:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499680 x1874672085751808/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829113 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829088.777416:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829088.777426:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:3.0:1787829088.777446:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649bc00 x1874672085752192/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829088.777448:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829088.777474:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:3.0:1787829088.777485:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787829088.777494:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085751808:192.168.203.141@tcp:8:kworker.0 00000100:00080000:3.0:1787829088.777496:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787829088.777504:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829088.777513:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2524 00000100:00100000:0.0:1787829088.777515:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085751936:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1787829088.777523:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829088.777530:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829088.777537:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:3.0:1787829088.777557:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498b40 x1874672085752320/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829088.777558:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829088.777577:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829088/real 1787829088] req@ffff9e35c6498f00 x1874672085751936/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829113 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829088.777591:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498f00 x1874672085751936/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829113 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829088.777621:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829088.777625:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829088.777630:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085751936:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829088.777670:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085752064:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829088.777687:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829088.777699:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829088/real 1787829088] req@ffff9e35c649b840 x1874672085752064/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829113 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829088.777718:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b840 x1874672085752064/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829113 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829088.777728:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829088.777748:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829088.777752:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085752064:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829088.777762:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085752192:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829088.777782:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829088.777791:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829088/real 1787829088] req@ffff9e35c649bc00 x1874672085752192/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829113 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829088.777801:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649bc00 x1874672085752192/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829113 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829088.777808:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829088.777811:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829088.777814:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085752192:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829088.777819:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085752320:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829088.777858:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829088.777867:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829088/real 1787829088] req@ffff9e35c6498b40 x1874672085752320/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829113 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829088.777877:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498b40 x1874672085752320/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829113 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829088.777905:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829088.777908:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829088.777910:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085752320:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829088.903933:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829088.903941:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829088.903995:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787829088.904047:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829089.927923:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829089.927933:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829089.927939:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787829089.927976:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829090.951918:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829090.951939:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829090.951939:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829090.951944:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829091.975862:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829091.975866:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829091.975936:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787829091.976013:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829092.999906:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829092.999944:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829092.999944:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787829093.000057:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829093.895937:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829093.895948:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829093.895954:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829093.895966:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2529 00000100:00080000:1.0:1787829093.895980:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829093.895990:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787829093.896046:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6d00 x1874672085752448/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829093.896076:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829093.896079:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829093.896081:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829093.896084:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2529 00000100:00080000:1.0:1787829093.896088:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829093.896091:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829093.896102:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829093.896113:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6940 x1874672085752576/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829093.896122:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829093.896125:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829093.896127:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829093.896132:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2529 00000100:00080000:1.0:1787829093.896138:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787829093.896140:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085752448:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787829093.896143:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829093.896159:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829093.896169:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6580 x1874672085752704/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829093.896182:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829093.896185:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829093.896187:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829093.896191:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2529 00000100:00080000:1.0:1787829093.896209:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829093.896216:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:0.0:1787829093.896221:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:1.0:1787829093.896232:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:1.0:1787829093.896259:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f43c0 x1874672085752832/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829093.896266:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829093/real 1787829093] req@ffff9e35da6f6d00 x1874672085752448/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829118 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829093.896281:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829093.896291:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829093.896293:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829093.896298:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2529 00000100:00100000:0.0:1787829093.896303:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6d00 x1874672085752448/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829118 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829093.896309:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829093.896317:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829093.896319:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:0.0:1787829093.896324:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829093.896331:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085752448:192.168.203.141@tcp:250:kworker.0 00000008:00100000:1.0:1787829093.896333:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:1.0:1787829093.896356:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f61c0 x1874672085752960/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829093.896362:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085752576:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787829093.896371:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829093.896374:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829093.896376:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829093.896382:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2529 00000100:00080000:1.0:1787829093.896392:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829093.896399:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:0.0:1787829093.896403:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:1.0:1787829093.896406:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829093.896420:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4f00 x1874672085753088/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829093.896430:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829093/real 1787829093] req@ffff9e35da6f6940 x1874672085752576/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829118 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829093.896430:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829093.896444:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829093.896449:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829093.896463:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2529 00000100:00080000:1.0:1787829093.896471:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829093.896480:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829093.896488:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829093.896506:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6d00 x1874672085753216/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829093.896509:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6940 x1874672085752576/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829118 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829093.896556:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829093.896559:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829093.896560:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829093.896581:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829093.896594:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:1.0:1787829093.896594:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2529 00000100:00080000:1.0:1787829093.896605:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787829093.896608:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085752576:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787829093.896616:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787829093.896622:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085752704:192.168.203.141@tcp:38:kworker.0 00000008:00100000:1.0:1787829093.896623:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:1.0:1787829093.896646:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4780 x1874672085753344/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829093.896661:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787829093.896666:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829093.896671:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829093.896673:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787829093.896678:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829093/real 1787829093] req@ffff9e35da6f6580 x1874672085752704/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829118 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829093.896681:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2529 00000100:00080000:1.0:1787829093.896691:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829093.896702:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829093.896715:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:0.0:1787829093.896724:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6580 x1874672085752704/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829118 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829093.896733:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7480 x1874672085753472/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829093.896736:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829093.896751:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829093.896754:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085752704:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829093.896771:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085752832:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829093.896789:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829093.896839:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829093/real 1787829093] req@ffff9e35da6f43c0 x1874672085752832/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829118 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829093.896848:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f43c0 x1874672085752832/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829118 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829093.896858:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829093.896860:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829093.896863:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085752832:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829093.896878:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085752960:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829093.896892:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829093.896899:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829093/real 1787829093] req@ffff9e35da6f61c0 x1874672085752960/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829118 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829093.896906:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f61c0 x1874672085752960/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829118 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829093.896912:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829093.896914:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829093.896917:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085752960:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829093.896924:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085753088:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829093.896943:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829093.896955:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829093/real 1787829093] req@ffff9e35da6f4f00 x1874672085753088/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829118 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829093.896968:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4f00 x1874672085753088/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829118 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829093.896977:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829093.896980:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829093.896982:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085753088:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829093.896987:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085753216:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829093.897002:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829093.897010:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829093/real 1787829093] req@ffff9e35da6f6d00 x1874672085753216/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829118 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829093.897017:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6d00 x1874672085753216/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829118 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829093.897023:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829093.897025:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829093.897027:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085753216:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829093.897031:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085753344:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829093.897041:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829093.897047:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829093/real 1787829093] req@ffff9e35da6f4780 x1874672085753344/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829118 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829093.897054:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4780 x1874672085753344/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829118 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829093.897063:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829093.897066:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829093.897070:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085753344:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829093.897076:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085753472:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829093.897096:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829093.897109:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829093/real 1787829093] req@ffff9e35da6f7480 x1874672085753472/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829118 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829093.897120:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7480 x1874672085753472/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829118 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829093.897129:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829093.897132:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829093.897135:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085753472:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829094.023867:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829094.023899:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829094.023964:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829094.023981:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829095.047914:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829095.047914:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829095.047932:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829095.047947:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829096.071904:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829096.071940:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:1.0:1787829096.071957:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829096.071968:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829097.095904:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829097.095904:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829097.095963:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829097.095973:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829098.119936:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829098.119937:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829098.119961:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829098.119978:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00080000:2.0:1787829099.015916:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829099.015925:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829099.015929:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829099.015937:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2534 00000100:00080000:2.0:1787829099.015946:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829099.015952:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787829099.015986:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498d680 x1874672085753600/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829099.016008:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829099.016014:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829099.016015:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829099.016018:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2534 00000100:00080000:2.0:1787829099.016020:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829099.016022:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829099.016031:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829099.016039:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c780 x1874672085753728/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829099.016047:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829099.016050:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829099.016052:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829099.016057:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2534 00000100:00100000:0.0:1787829099.016058:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085753600:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1787829099.016060:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829099.016065:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829099.016071:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829099.016078:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e940 x1874672085753856/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829099.016085:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829099.016088:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829099.016089:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829099.016092:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2534 00000100:00080000:2.0:1787829099.016094:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829099.016096:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829099.016100:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:2.0:1787829099.016108:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498fc00 x1874672085753984/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829099.016114:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829099.016116:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829099.016118:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829099.016120:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2534 00000100:00080000:2.0:1787829099.016123:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829099.016125:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829099.016128:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000800:00000100:0.0:1787829099.016133:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829099.016138:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498de00 x1874672085754112/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829099.016145:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829099.016148:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829099.016149:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829099.016152:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2534 00000100:00080000:2.0:1787829099.016155:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829099.016158:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829099.016160:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829099.016168:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498ed00 x1874672085754240/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829099.016174:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829099.016176:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829099.016178:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829099.016183:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2534 00000100:00100000:0.0:1787829099.016184:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829099/real 1787829099] req@ffff9e35c498d680 x1874672085753600/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829124 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829099.016186:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829099.016192:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829099.016197:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829099.016208:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c3c0 x1874672085754368/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829099.016215:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787829099.016216:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498d680 x1874672085753600/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829124 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829099.016221:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829099.016225:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829099.016235:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2534 00000100:00080000:0.0:1787829099.016240:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829099.016241:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829099.016249:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:2.0:1787829099.016251:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787829099.016259:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085753600:192.168.203.141@tcp:250:kworker.0 00000008:00100000:2.0:1787829099.016260:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:2.0:1787829099.016280:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cf00 x1874672085754496/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829099.016291:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085753728:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1787829099.016292:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829099.016302:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829099.016304:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829099.016310:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2534 00000100:00080000:2.0:1787829099.016315:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829099.016320:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:0.0:1787829099.016326:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:2.0:1787829099.016326:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:2.0:1787829099.016346:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c000 x1874672085754624/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829099.016347:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829099/real 1787829099] req@ffff9e35c498c780 x1874672085753728/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829124 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829099.016383:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c780 x1874672085753728/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829124 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829099.016394:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829099.016399:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829099.016403:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085753728:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829099.016411:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085753856:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829099.016431:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829099.016442:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829099/real 1787829099] req@ffff9e35c498e940 x1874672085753856/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829124 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829099.016454:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e940 x1874672085753856/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829124 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829099.016464:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829099.016467:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829099.016470:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085753856:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829099.016477:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085753984:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829099.016494:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829099.016505:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829099/real 1787829099] req@ffff9e35c498fc00 x1874672085753984/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829124 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829099.016517:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498fc00 x1874672085753984/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829124 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829099.016527:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829099.016530:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829099.016550:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085753984:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829099.016556:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085754112:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829099.016576:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829099.016587:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829099/real 1787829099] req@ffff9e35c498de00 x1874672085754112/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829124 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829099.016599:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498de00 x1874672085754112/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829124 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829099.016609:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829099.016612:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829099.016616:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085754112:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829099.016622:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085754240:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829099.016642:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829099.016654:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829099/real 1787829099] req@ffff9e35c498ed00 x1874672085754240/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829124 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829099.016667:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498ed00 x1874672085754240/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829124 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829099.016676:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829099.016679:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829099.016683:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085754240:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829099.016690:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085754368:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829099.016708:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829099.016719:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829099/real 1787829099] req@ffff9e35c498c3c0 x1874672085754368/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829124 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829099.016732:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c3c0 x1874672085754368/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829124 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829099.016741:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829099.016744:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829099.016748:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085754368:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829099.016772:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085754496:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829099.016792:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829099.016834:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829099/real 1787829099] req@ffff9e35c498cf00 x1874672085754496/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829124 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829099.016847:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498cf00 x1874672085754496/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829124 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829099.016857:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829099.016860:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829099.016864:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085754496:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829099.016871:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085754624:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829099.016891:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829099.016903:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829099/real 1787829099] req@ffff9e35c498c000 x1874672085754624/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829124 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829099.016916:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c000 x1874672085754624/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829124 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829099.016925:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829099.016928:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829099.016931:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085754624:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829099.143901:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829099.143901:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829099.143904:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787829099.143958:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829100.167897:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829100.167897:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829100.167913:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829100.167941:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829101.191878:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829101.191879:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829101.191890:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829101.191954:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829102.215898:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829102.215898:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829102.215900:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:1.0:1787829102.215954:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829103.239884:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829103.239900:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829103.239925:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829103.239945:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829104.135886:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829104.135900:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829104.135904:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829104.135913:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2539 00000100:00080000:2.0:1787829104.135921:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829104.135928:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787829104.135964:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cb40 x1874672085754752/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829104.135986:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829104.135989:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829104.135991:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829104.135994:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2539 00000100:00080000:2.0:1787829104.135996:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829104.135999:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829104.136007:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829104.136019:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f840 x1874672085754880/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829104.136035:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085754752:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1787829104.136036:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829104.136042:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829104.136046:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829104.136050:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2539 00000100:00080000:2.0:1787829104.136053:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829104.136056:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829104.136060:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829104.136071:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e580 x1874672085755008/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829104.136078:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829104.136081:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829104.136082:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829104.136085:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2539 00000100:00080000:2.0:1787829104.136088:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829104.136091:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829104.136095:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:2.0:1787829104.136103:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f0c0 x1874672085755136/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829104.136109:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1787829104.136115:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829104.136118:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829104.136120:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829104.136124:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2539 00000100:00080000:2.0:1787829104.136129:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829104.136133:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829104.136137:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:2.0:1787829104.136149:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f480 x1874672085755264/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829104.136159:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787829104.136162:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829104/real 1787829104] req@ffff9e35c498cb40 x1874672085754752/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829129 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829104.136165:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829104.136170:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829104.136181:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2539 00000100:00080000:2.0:1787829104.136187:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829104.136194:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787829104.136195:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498cb40 x1874672085754752/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829129 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1787829104.136204:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1787829104.136210:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829104.136218:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829104.136223:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e1c0 x1874672085755392/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829104.136225:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085754752:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1787829104.136242:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829104.136247:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829104.136249:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829104.136254:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2539 00000100:00080000:2.0:1787829104.136259:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787829104.136260:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085754880:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1787829104.136269:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829104.136277:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1787829104.136304:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829104.136304:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cb40 x1874672085755520/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829104.136318:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829104/real 1787829104] req@ffff9e35c498f840 x1874672085754880/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829129 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829104.136348:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787829104.136349:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f840 x1874672085754880/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829129 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829104.136360:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829104.136364:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829104.136372:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829104.136377:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2539 00000100:00080000:0.0:1787829104.136382:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:2.0:1787829104.136383:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787829104.136390:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085754880:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1787829104.136391:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829104.136400:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:0.0:1787829104.136412:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085755008:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829104.136415:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c000 x1874672085755648/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829104.136424:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829104.136427:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829104.136429:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829104.136433:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2539 00000100:00080000:2.0:1787829104.136437:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829104.136440:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:0.0:1787829104.136445:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:2.0:1787829104.136447:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:2.0:1787829104.136461:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cf00 x1874672085755776/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829104.136462:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829104/real 1787829104] req@ffff9e35c498e580 x1874672085755008/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829129 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829104.136490:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e580 x1874672085755008/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829129 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829104.136511:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829104.136515:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829104.136518:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085755008:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829104.136524:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085755136:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829104.136542:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829104.136552:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829104/real 1787829104] req@ffff9e35c498f0c0 x1874672085755136/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829129 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829104.136560:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f0c0 x1874672085755136/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829129 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829104.136570:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829104.136572:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829104.136574:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085755136:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829104.136578:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085755264:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829104.136588:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829104.136593:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829104/real 1787829104] req@ffff9e35c498f480 x1874672085755264/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829129 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829104.136600:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f480 x1874672085755264/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829129 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829104.136606:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829104.136608:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829104.136610:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085755264:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829104.136613:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085755392:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829104.136623:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829104.136629:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829104/real 1787829104] req@ffff9e35c498e1c0 x1874672085755392/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829129 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829104.136636:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e1c0 x1874672085755392/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829129 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829104.136642:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829104.136643:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829104.136646:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085755392:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829104.136666:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085755520:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829104.136688:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829104.136700:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829104/real 1787829104] req@ffff9e35c498cb40 x1874672085755520/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829129 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829104.136712:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498cb40 x1874672085755520/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829129 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829104.136718:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829104.136720:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829104.136722:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085755520:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829104.136726:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085755648:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829104.136740:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829104.136746:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829104/real 1787829104] req@ffff9e35c498c000 x1874672085755648/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829129 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829104.136762:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c000 x1874672085755648/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829129 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829104.136771:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829104.136774:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829104.136777:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085755648:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829104.136784:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085755776:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829104.136830:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829104.136849:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829104/real 1787829104] req@ffff9e35c498cf00 x1874672085755776/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829129 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829104.136858:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498cf00 x1874672085755776/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829129 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829104.136864:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829104.136866:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829104.136868:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085755776:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829104.263906:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829104.263908:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829104.263949:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787829104.263954:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829105.287903:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829105.287925:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829105.287927:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:1.0:1787829105.287985:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829106.311903:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829106.311903:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829106.311925:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829106.311953:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829107.335911:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829107.335917:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829107.335925:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787829107.337179:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829108.359929:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829108.359929:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829108.359952:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787829108.359958:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829109.256050:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829109.256058:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787829109.256062:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829109.256071:0:27606:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2544 00000100:00080000:3.0:1787829109.256100:0:27606:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829109.256106:0:27606:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787829109.256138:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a940 x1874672085755904/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829109.256161:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829109.256164:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829109.256165:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829109.256168:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2544 00000100:00080000:3.0:1787829109.256171:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829109.256174:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829109.256182:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829109.256190:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498780 x1874672085756032/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829109.256197:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829109.256199:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787829109.256200:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829109.256203:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2544 00000100:00080000:3.0:1787829109.256205:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829109.256207:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829109.256209:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829109.256216:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499a40 x1874672085756160/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829109.256222:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829109.256225:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787829109.256227:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829109.256230:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2544 00000100:00080000:3.0:1787829109.256232:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829109.256234:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829109.256238:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:3.0:1787829109.256247:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a1c0 x1874672085756288/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829109.256261:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085755904:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1787829109.256262:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829109.256272:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787829109.256279:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829109.256283:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2544 00000100:00080000:3.0:1787829109.256285:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829109.256287:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829109.256291:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:3.0:1787829109.256308:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a580 x1874672085756416/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829109.256318:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829109.256322:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829109.256325:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829109.256329:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2544 00000100:00080000:3.0:1787829109.256333:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829109.256338:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829109.256343:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829109.256355:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b0c0 x1874672085756544/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829109.256365:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829109.256367:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787829109.256369:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829109.256371:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2544 00000100:00080000:3.0:1787829109.256375:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829109.256377:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829109.256381:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829109.256389:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b480 x1874672085756672/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829109.256394:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829109.256396:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787829109.256397:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829109.256400:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2544 00000100:00080000:3.0:1787829109.256402:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829109.256404:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829109.256406:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:3.0:1787829109.256418:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c64992c0 x1874672085756800/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829109.256420:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1787829109.256433:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829109.256436:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787829109.256437:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829109.256444:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2544 00000100:00080000:3.0:1787829109.256446:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829109.256449:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829109.256453:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:3.0:1787829109.256478:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499e00 x1874672085756928/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829109.256495:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829109/real 1787829109] req@ffff9e35c649a940 x1874672085755904/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829134 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829109.256514:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a940 x1874672085755904/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829134 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829109.256525:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829109.256529:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829109.256532:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085755904:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1787829109.256539:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085756032:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829109.256562:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829109.256572:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829109/real 1787829109] req@ffff9e35c6498780 x1874672085756032/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829134 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829109.256580:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498780 x1874672085756032/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829134 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829109.256586:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829109.256589:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829109.256591:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085756032:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829109.256595:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085756160:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829109.256605:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829109.256611:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829109/real 1787829109] req@ffff9e35c6499a40 x1874672085756160/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829134 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829109.256618:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499a40 x1874672085756160/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829134 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829109.256624:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829109.256626:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829109.256627:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085756160:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829109.256652:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085756288:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829109.256671:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829109.256679:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829109/real 1787829109] req@ffff9e35c649a1c0 x1874672085756288/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829134 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829109.256699:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a1c0 x1874672085756288/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829134 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829109.256708:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829109.256712:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829109.256715:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085756288:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829109.256722:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085756416:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829109.256750:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829109.256759:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829109/real 1787829109] req@ffff9e35c649a580 x1874672085756416/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829134 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829109.256767:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a580 x1874672085756416/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829134 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829109.256773:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829109.256776:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829109.256778:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085756416:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829109.256783:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085756544:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829109.256794:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829109.257962:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829109/real 1787829109] req@ffff9e35c649b0c0 x1874672085756544/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829134 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829109.257973:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b0c0 x1874672085756544/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829134 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829109.257985:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829109.257987:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829109.257990:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085756544:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829109.257994:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085756672:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829109.258012:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829109.258019:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829109/real 1787829109] req@ffff9e35c649b480 x1874672085756672/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829134 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829109.258027:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b480 x1874672085756672/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829134 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829109.258032:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829109.258034:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829109.258036:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085756672:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829109.258040:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c64992c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085756800:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829109.258051:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829109.258057:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829109/real 1787829109] req@ffff9e35c64992c0 x1874672085756800/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829134 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829109.258065:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c64992c0 x1874672085756800/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829134 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829109.258070:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829109.258072:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829109.258075:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c64992c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085756800:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829109.258079:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085756928:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829109.258089:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829109.258094:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829109/real 1787829109] req@ffff9e35c6499e00 x1874672085756928/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829134 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829109.258102:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499e00 x1874672085756928/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829134 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829109.258108:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829109.258109:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829109.258111:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085756928:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829109.383884:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829109.383896:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829109.383933:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829109.383978:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829110.407909:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829110.407909:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787829110.408053:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:0.0:1787829110.408129:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829111.431888:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829111.431908:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829111.431930:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829111.431938:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829112.455904:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829112.455944:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829112.455992:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829112.455999:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829113.479917:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829113.479918:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787829113.479989:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829113.480010:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00080000:3.0:1787829114.375933:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829114.375944:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787829114.375948:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829114.375960:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2549 00000100:00080000:3.0:1787829114.375972:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829114.375980:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787829114.376022:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c64983c0 x1874672085757056/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829114.376051:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829114.376054:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829114.376056:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829114.376059:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2549 00000100:00080000:3.0:1787829114.376062:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829114.376064:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829114.376075:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829114.376084:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649ad00 x1874672085757184/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829114.376092:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829114.376096:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787829114.376100:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787829114.376101:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c64983c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085757056:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1787829114.376105:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2549 00000100:00080000:3.0:1787829114.376110:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829114.376113:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829114.376116:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829114.376124:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498000 x1874672085757312/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829114.376130:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829114.376133:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787829114.376134:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829114.376137:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2549 00000100:00080000:3.0:1787829114.376139:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829114.376142:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829114.376145:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:3.0:1787829114.376154:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499e00 x1874672085757440/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829114.376160:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829114.376163:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787829114.376164:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829114.376168:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2549 00000100:00080000:3.0:1787829114.376170:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829114.376172:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829114.376175:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000800:00000100:0.0:1787829114.376180:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787829114.376188:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c64992c0 x1874672085757568/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829114.376197:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829114.376200:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829114.376201:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829114.376204:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2549 00000100:00080000:3.0:1787829114.376206:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829114.376208:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829114.376211:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829114.376219:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b480 x1874672085757696/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829114.376226:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829114.376230:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:0.0:1787829114.376231:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829114/real 1787829114] req@ffff9e35c64983c0 x1874672085757056/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829139 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829114.376232:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829114.376239:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2549 00000100:00080000:3.0:1787829114.376243:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829114.376249:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829114.376255:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829114.376264:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b0c0 x1874672085757824/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829114.376270:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c64983c0 x1874672085757056/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829139 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829114.376273:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829114.376279:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787829114.376282:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829114.376290:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2549 00000100:00080000:0.0:1787829114.376294:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829114.376294:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829114.376298:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:0.0:1787829114.376303:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000008:00100000:3.0:1787829114.376304:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:0.0:1787829114.376309:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c64983c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085757056:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1787829114.376314:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a580 x1874672085757952/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829114.376321:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829114.376323:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787829114.376325:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829114.376328:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2549 00000100:00080000:3.0:1787829114.376330:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829114.376333:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829114.376336:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:0.0:1787829114.376337:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085757184:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787829114.376346:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a1c0 x1874672085758080/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829114.376366:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829114.376378:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829114/real 1787829114] req@ffff9e35c649ad00 x1874672085757184/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829139 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829114.376388:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649ad00 x1874672085757184/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829139 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829114.376394:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829114.376396:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829114.376399:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085757184:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829114.376403:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085757312:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829114.376414:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829114.376421:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829114/real 1787829114] req@ffff9e35c6498000 x1874672085757312/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829139 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829114.376447:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498000 x1874672085757312/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829139 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829114.376453:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829114.376456:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829114.376458:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085757312:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829114.376462:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085757440:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829114.376477:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829114.376484:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829114/real 1787829114] req@ffff9e35c6499e00 x1874672085757440/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829139 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829114.376497:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499e00 x1874672085757440/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829139 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829114.376502:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829114.376504:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829114.376506:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085757440:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829114.376509:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c64992c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085757568:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829114.376520:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829114.376525:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829114/real 1787829114] req@ffff9e35c64992c0 x1874672085757568/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829139 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829114.376533:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c64992c0 x1874672085757568/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829139 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829114.376539:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829114.376540:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829114.376542:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c64992c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085757568:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829114.376546:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085757696:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829114.376556:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829114.376562:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829114/real 1787829114] req@ffff9e35c649b480 x1874672085757696/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829139 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829114.376577:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b480 x1874672085757696/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829139 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829114.376587:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829114.376589:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829114.376591:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085757696:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829114.376595:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085757824:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829114.376606:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829114.376612:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829114/real 1787829114] req@ffff9e35c649b0c0 x1874672085757824/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829139 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829114.376620:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b0c0 x1874672085757824/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829139 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829114.376625:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829114.376627:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829114.376628:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085757824:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829114.376632:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085757952:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829114.376642:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829114.376648:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829114/real 1787829114] req@ffff9e35c649a580 x1874672085757952/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829139 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829114.376655:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a580 x1874672085757952/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829139 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829114.376660:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829114.376662:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829114.376665:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085757952:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829114.376683:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085758080:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829114.376701:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829114.376709:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829114/real 1787829114] req@ffff9e35c649a1c0 x1874672085758080/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829139 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829114.376717:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a1c0 x1874672085758080/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829139 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829114.376723:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829114.376725:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829114.376727:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085758080:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829114.503951:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787829114.503951:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829114.503956:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829114.504050:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829115.527901:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829115.527901:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829115.527933:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829115.527959:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829116.551900:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829116.551911:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829116.551935:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787829116.551943:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829117.575923:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829117.575957:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829117.575958:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829117.575976:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829118.599936:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829118.599939:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829118.599944:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787829118.600015:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829119.495903:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829119.495910:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787829119.495915:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829119.495926:0:27606:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2554 00000100:00080000:0.0:1787829119.495937:0:27606:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829119.495944:0:27606:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787829119.495981:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718b40 x1874672085758208/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829119.496007:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829119.496012:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829119.496014:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829119.496016:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2554 00000100:00080000:0.0:1787829119.496019:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829119.496022:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829119.496030:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829119.496042:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718f00 x1874672085758336/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829119.496051:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829119.496053:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787829119.496054:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829119.496060:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2554 00000100:00100000:2.0:1787829119.496061:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085758208:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1787829119.496062:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829119.496068:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829119.496073:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829119.496080:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a940 x1874672085758464/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829119.496086:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829119.496088:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829119.496090:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829119.496092:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2554 00000100:00080000:0.0:1787829119.496094:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829119.496097:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829119.496101:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:0.0:1787829119.496109:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7183c0 x1874672085758592/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829119.496115:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829119.496117:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829119.496119:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829119.496122:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2554 00000100:00080000:0.0:1787829119.496124:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829119.496126:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829119.496129:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:0.0:1787829119.496136:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a1c0 x1874672085758720/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829119.496142:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829119.496145:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829119.496146:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829119.496148:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2554 00000100:00080000:0.0:1787829119.496150:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829119.496153:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829119.496156:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829119.496166:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718780 x1874672085758848/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829119.496172:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829119.496173:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787829119.496176:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829119.496178:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2554 00000100:00080000:0.0:1787829119.496181:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787829119.496183:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085758336:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787829119.496186:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829119.496193:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829119.496204:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da719a40 x1874672085758976/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829119.496211:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829119.496213:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829119.496215:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829119.496218:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2554 00000100:00080000:0.0:1787829119.496220:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829119.496223:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829119.496227:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:2.0:1787829119.496231:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085758464:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829119.496239:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a580 x1874672085759104/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829119.496246:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829119.496248:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829119.496250:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829119.496252:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2554 00000100:00080000:0.0:1787829119.496257:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829119.496260:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829119.496262:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:0.0:1787829119.496274:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718000 x1874672085759232/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829119.496274:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085758592:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829119.496314:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085758720:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829119.496336:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085758848:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829119.496354:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085758976:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829119.496365:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085759104:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829119.496375:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085759232:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829119.623895:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829119.623895:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829119.623958:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829119.623964:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829120.647896:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829120.647896:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829120.647945:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829120.647989:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829121.671955:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829121.671955:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829121.671969:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829121.671993:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1787829122.568078:0:2398:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.141:988 00000400:00000100:3.0:1787829122.568124:0:2398:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.141@tcp at host 192.168.203.141 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1787829122.568142:0:2398:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1787829122.568224:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000800:00000100:3.0:1787829122.568245:0:2398:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:2.0:1787829122.568247:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829119/real 1787829122] req@ffff9e35da718b40 x1874672085758208/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829144 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:3.0:1787829122.568258:0:2398:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1787829122.568273:0:2398:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:2.0:1787829122.568290:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718b40 x1874672085758208/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829144 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:3.0:1787829122.568290:0:2398:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1787829122.568303:0:2398:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00080000:2.0:1787829122.568310:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000800:00000100:3.0:1787829122.568317:0:2398:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00080000:2.0:1787829122.568321:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829122.568330:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085758208:192.168.203.141@tcp:250:kworker.0 00000800:00000100:3.0:1787829122.568332:0:2398:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:2.0:1787829122.568363:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829119/real 1787829122] req@ffff9e35da719a40 x1874672085758976/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829144 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829122.568397:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672085758976/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829144 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829122.568407:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829122.568411:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829122.568415:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085758976:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829122.568423:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829119/real 1787829122] req@ffff9e35da71a580 x1874672085759104/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829144 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829122.568436:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672085759104/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829144 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829122.568446:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829122.568450:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829122.568454:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085759104:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829122.568462:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829119/real 1787829122] req@ffff9e35da718000 x1874672085759232/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829144 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829122.568475:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718000 x1874672085759232/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829144 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829122.568484:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829122.568487:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829122.568491:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085759232:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829122.568498:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829119/real 1787829122] req@ffff9e35da718f00 x1874672085758336/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829144 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829122.568511:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672085758336/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829144 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829122.568519:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829122.568524:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829122.568527:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085758336:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829122.568535:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829119/real 1787829122] req@ffff9e35da71a940 x1874672085758464/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829144 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829122.568547:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672085758464/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829144 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829122.568556:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829122.568560:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829122.568564:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085758464:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829122.568572:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829119/real 1787829122] req@ffff9e35da7183c0 x1874672085758592/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829144 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829122.568583:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672085758592/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829144 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829122.568591:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829122.568595:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829122.568598:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085758592:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829122.568605:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829119/real 1787829122] req@ffff9e35da71a1c0 x1874672085758720/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829144 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829122.568617:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a1c0 x1874672085758720/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829144 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829122.568627:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829122.568630:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829122.568634:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085758720:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829122.568641:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829119/real 1787829122] req@ffff9e35da718780 x1874672085758848/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829144 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829122.568654:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718780 x1874672085758848/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829144 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829122.568663:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829122.568667:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829122.568670:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085758848:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829122.695886:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829122.695889:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829122.695909:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:1.0:1787829122.695945:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829123.719889:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829123.719893:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829123.719923:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829123.719934:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00080000:1.0:1787829124.615909:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829124.615920:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829124.615924:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829124.615934:0:27606:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2559 00000100:00080000:1.0:1787829124.615944:0:27606:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829124.615951:0:27606:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787829124.615985:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5e00 x1874672085759360/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829124.616009:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829124.616012:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829124.616014:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829124.616017:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2559 00000100:00080000:1.0:1787829124.616019:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829124.616022:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829124.616031:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829124.616040:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4b40 x1874672085759488/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829124.616049:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829124.616054:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:2.0:1787829124.616055:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085759360:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787829124.616057:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829124.616066:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2559 00000100:00080000:1.0:1787829124.616068:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829124.616070:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829124.616073:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829124.616083:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f70c0 x1874672085759616/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829124.616090:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829124.616092:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829124.616093:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829124.616096:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2559 00000100:00080000:1.0:1787829124.616098:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829124.616101:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829124.616104:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:1.0:1787829124.616112:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7840 x1874672085759744/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829124.616117:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829124.616119:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829124.616122:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1787829124.616124:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787829124.616127:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2559 00000100:00080000:1.0:1787829124.616132:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829124.616134:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829124.616138:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:1.0:1787829124.616147:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5a40 x1874672085759872/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829124.616152:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829124.616155:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829124.616156:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829124.616159:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2559 00000100:00080000:1.0:1787829124.616162:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829124.616164:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829124.616168:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829124.616172:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829124/real 1787829124] req@ffff9e35da6f5e00 x1874672085759360/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829149 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829124.616178:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4000 x1874672085760000/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829124.616203:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829124.616205:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829124.616207:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829124.616210:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2559 00000100:00080000:1.0:1787829124.616214:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787829124.616218:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5e00 x1874672085759360/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829149 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829124.616221:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829124.616229:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1787829124.616240:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829124.616247:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829124.616255:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085759360:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1787829124.616256:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7480 x1874672085760128/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829124.616270:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829124.616274:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829124.616276:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829124.616281:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2559 00000100:00080000:1.0:1787829124.616285:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829124.616289:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787829124.616294:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085759488:192.168.203.141@tcp:38:kworker.0 00000008:00100000:1.0:1787829124.616297:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:1.0:1787829124.616324:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4780 x1874672085760256/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829124.616331:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787829124.616343:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829124.616347:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829124.616349:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787829124.616352:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829124/real 1787829124] req@ffff9e35da6f4b40 x1874672085759488/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829149 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829124.616371:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2559 00000100:00080000:1.0:1787829124.616379:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829124.616386:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787829124.616391:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4b40 x1874672085759488/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829149 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1787829124.616393:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00080000:2.0:1787829124.616407:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829124.616411:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829124.616414:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085759488:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829124.616422:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085759616:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829124.616427:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6d00 x1874672085760384/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829124.616455:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829124.616469:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829124/real 1787829124] req@ffff9e35da6f70c0 x1874672085759616/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829149 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829124.616482:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f70c0 x1874672085759616/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829149 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829124.616491:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829124.616494:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829124.616498:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085759616:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829124.616504:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085759744:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829124.616525:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829124.616537:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829124/real 1787829124] req@ffff9e35da6f7840 x1874672085759744/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829149 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829124.616550:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7840 x1874672085759744/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829149 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829124.616557:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829124.616559:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829124.616562:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085759744:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829124.616568:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085759872:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829124.616589:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829124.616600:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829124/real 1787829124] req@ffff9e35da6f5a40 x1874672085759872/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829149 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829124.616619:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5a40 x1874672085759872/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829149 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829124.616627:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829124.616630:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829124.616633:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085759872:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829124.616640:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085760000:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787829124.616660:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829124.616672:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829124/real 1787829124] req@ffff9e35da6f4000 x1874672085760000/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829149 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829124.616684:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4000 x1874672085760000/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829149 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829124.616691:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829124.616694:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829124.616696:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085760000:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829124.616700:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085760128:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787829124.616720:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829124.616731:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829124/real 1787829124] req@ffff9e35da6f7480 x1874672085760128/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829149 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829124.616744:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7480 x1874672085760128/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829149 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829124.616758:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829124.616761:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829124.616764:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085760128:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829124.616793:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085760256:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829124.616861:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829124.616873:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829124/real 1787829124] req@ffff9e35da6f4780 x1874672085760256/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829149 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829124.616881:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4780 x1874672085760256/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829149 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829124.616891:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829124.616894:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829124.616898:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085760256:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829124.616904:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085760384:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829124.616923:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829124.616935:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829124/real 1787829124] req@ffff9e35da6f6d00 x1874672085760384/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829149 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829124.616948:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6d00 x1874672085760384/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829149 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829124.616955:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829124.616957:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829124.616960:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085760384:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829124.743913:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829124.743914:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829124.743933:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829124.743945:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829125.767916:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829125.767928:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787829125.768022:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829125.768066:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829126.791901:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829126.791902:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:3.0:1787829126.791904:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829126.791973:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829127.815917:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829127.815924:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829127.815983:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829127.815993:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829128.839868:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829128.839876:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829128.839889:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787829128.839942:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829129.735899:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829129.735907:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829129.735910:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829129.735918:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2565 00000100:00080000:1.0:1787829129.735927:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829129.735933:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787829129.735965:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4f00 x1874672085760512/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829129.735990:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829129.735993:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829129.735995:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829129.735998:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2565 00000100:00080000:1.0:1787829129.736000:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829129.736002:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829129.736010:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829129.736019:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f61c0 x1874672085760640/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829129.736043:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085760512:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787829129.736076:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829129.736079:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829129.736081:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829129.736084:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2565 00000100:00080000:1.0:1787829129.736087:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829129.736089:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829129.736093:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829129.736102:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f43c0 x1874672085760768/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829129.736109:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829129.736112:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829129.736113:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829129.736116:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2565 00000100:00080000:1.0:1787829129.736118:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1787829129.736124:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787829129.736124:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829129.736131:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:1.0:1787829129.736140:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6580 x1874672085760896/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829129.736148:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829129.736151:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829129.736152:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829129.736155:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2565 00000100:00080000:1.0:1787829129.736158:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829129.736161:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829129.736164:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:1.0:1787829129.736172:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6940 x1874672085761024/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829129.736176:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829129/real 1787829129] req@ffff9e35da6f4f00 x1874672085760512/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829154 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829129.736180:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829129.736187:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829129.736191:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829129.736199:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2565 00000100:00080000:1.0:1787829129.736202:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829129.736204:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829129.736207:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829129.736213:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4f00 x1874672085760512/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829154 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829129.736216:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6d00 x1874672085761152/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829129.736246:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829129.736249:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829129.736249:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829129.736252:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829129.736260:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2565 00000100:00080000:0.0:1787829129.736262:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:1.0:1787829129.736263:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829129.736268:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787829129.736273:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085760512:192.168.203.141@tcp:250:kworker.0 00000008:00100000:1.0:1787829129.736281:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829129.736294:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4780 x1874672085761280/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829129.736304:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085760640:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787829129.736309:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829129.736314:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829129.736316:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829129.736332:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2565 00000100:00080000:1.0:1787829129.736337:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829129.736342:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829129.736347:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:1.0:1787829129.736357:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7480 x1874672085761408/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829129.736366:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787829129.736367:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829129.736373:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829129.736376:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829129.736382:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2565 00000100:00100000:0.0:1787829129.736388:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829129/real 1787829129] req@ffff9e35da6f61c0 x1874672085760640/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829154 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829129.736388:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829129.736398:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829129.736407:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:0.0:1787829129.736424:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f61c0 x1874672085760640/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829154 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829129.736426:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4000 x1874672085761536/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829129.736460:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829129.736466:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829129.736471:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085760640:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829129.736479:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085760768:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829129.736501:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829129.736513:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829129/real 1787829129] req@ffff9e35da6f43c0 x1874672085760768/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829154 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829129.736526:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f43c0 x1874672085760768/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829154 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829129.736535:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829129.736539:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829129.736542:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085760768:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829129.736548:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085760896:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829129.736569:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829129.736591:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829129/real 1787829129] req@ffff9e35da6f6580 x1874672085760896/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829154 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829129.736603:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6580 x1874672085760896/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829154 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829129.736618:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829129.736622:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829129.736632:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085760896:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829129.736639:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085761024:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829129.736681:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829129.736697:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829129/real 1787829129] req@ffff9e35da6f6940 x1874672085761024/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829154 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829129.736721:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6940 x1874672085761024/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829154 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829129.736731:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829129.736735:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829129.736741:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085761024:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829129.736748:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085761152:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829129.736769:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829129.736789:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829129/real 1787829129] req@ffff9e35da6f6d00 x1874672085761152/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829154 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829129.736840:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6d00 x1874672085761152/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829154 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829129.736849:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829129.736852:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829129.736856:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085761152:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829129.736881:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085761280:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829129.736903:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829129.736915:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829129/real 1787829129] req@ffff9e35da6f4780 x1874672085761280/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829154 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829129.736937:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4780 x1874672085761280/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829154 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829129.736946:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829129.736950:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829129.736954:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085761280:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829129.736961:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085761408:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829129.736982:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829129.736994:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829129/real 1787829129] req@ffff9e35da6f7480 x1874672085761408/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829154 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829129.737006:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7480 x1874672085761408/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829154 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829129.737014:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829129.737018:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829129.737021:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085761408:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829129.737027:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085761536:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829129.737044:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829129.737055:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829129/real 1787829129] req@ffff9e35da6f4000 x1874672085761536/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829154 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829129.737067:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4000 x1874672085761536/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829154 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829129.737076:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829129.737079:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829129.737082:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085761536:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829129.863924:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829129.863924:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829129.863937:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829129.863967:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829130.887862:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829130.887863:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829130.887907:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829130.887933:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829131.911957:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829131.911957:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829131.912008:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787829131.912027:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829132.935905:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829132.935933:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829132.935946:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829132.935947:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829133.959950:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829133.959952:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829133.959954:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829133.960025:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829134.855895:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829134.855903:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829134.855907:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829134.855915:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2570 00000100:00080000:1.0:1787829134.855923:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829134.855929:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787829134.855959:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5a40 x1874672085761664/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829134.855983:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829134.855987:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829134.855989:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829134.855993:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2570 00000100:00080000:1.0:1787829134.855997:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829134.856000:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829134.856011:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829134.856026:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7840 x1874672085761792/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829134.856037:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829134.856040:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829134.856042:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829134.856047:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2570 00000100:00080000:1.0:1787829134.856050:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787829134.856073:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085761664:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787829134.856073:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829134.856087:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829134.856105:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f70c0 x1874672085761920/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829134.856119:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829134.856123:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829134.856125:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829134.856131:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2570 00000100:00080000:1.0:1787829134.856161:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829134.856166:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829134.856171:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000800:00000100:0.0:1787829134.856179:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829134.856188:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4b40 x1874672085762048/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829134.856196:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829134.856200:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829134.856201:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829134.856205:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2570 00000100:00080000:1.0:1787829134.856211:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829134.856214:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829134.856217:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:1.0:1787829134.856235:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5e00 x1874672085762176/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829134.856245:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829134.856249:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:0.0:1787829134.856250:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829134/real 1787829134] req@ffff9e35da6f5a40 x1874672085761664/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829159 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829134.856252:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829134.856261:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2570 00000100:00080000:1.0:1787829134.856266:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829134.856278:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829134.856297:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829134.856311:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4000 x1874672085762304/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829134.856321:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5a40 x1874672085761664/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829159 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829134.856324:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829134.856336:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829134.856339:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829134.856350:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2570 00000100:00080000:1.0:1787829134.856356:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829134.856361:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829134.856365:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1787829134.856366:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1787829134.856374:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7480 x1874672085762432/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829134.856376:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829134.856398:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085761664:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787829134.856399:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829134.856412:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829134.856414:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829134.856417:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2570 00000100:00080000:1.0:1787829134.856419:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829134.856429:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829134.856431:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:0.0:1787829134.856437:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085761792:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829134.856441:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4780 x1874672085762560/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829134.856458:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829134.856461:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829134.856462:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829134.856466:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2570 00000100:00080000:1.0:1787829134.856468:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829134.856470:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829134.856473:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:1.0:1787829134.856482:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6d00 x1874672085762688/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829134.856500:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829134.856523:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829134/real 1787829134] req@ffff9e35da6f7840 x1874672085761792/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829159 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829134.856548:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7840 x1874672085761792/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829159 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829134.856559:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829134.856564:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829134.856568:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085761792:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829134.856615:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085761920:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829134.856643:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829134.856657:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829134/real 1787829134] req@ffff9e35da6f70c0 x1874672085761920/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829159 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829134.856673:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f70c0 x1874672085761920/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829159 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829134.856683:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829134.856687:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829134.856691:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085761920:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829134.856699:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085762048:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829134.856721:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829134.856734:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829134/real 1787829134] req@ffff9e35da6f4b40 x1874672085762048/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829159 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829134.856759:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4b40 x1874672085762048/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829159 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829134.856776:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829134.856780:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829134.856784:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085762048:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829134.856791:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085762176:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829134.856868:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829134.856882:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829134/real 1787829134] req@ffff9e35da6f5e00 x1874672085762176/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829159 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829134.856896:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5e00 x1874672085762176/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829159 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829134.856906:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829134.856910:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829134.856914:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085762176:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829134.856921:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085762304:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829134.856943:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829134.856956:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829134/real 1787829134] req@ffff9e35da6f4000 x1874672085762304/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829159 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829134.856968:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4000 x1874672085762304/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829159 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829134.856978:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829134.856981:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829134.856986:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085762304:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829134.856993:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085762432:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829134.857013:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829134.857026:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829134/real 1787829134] req@ffff9e35da6f7480 x1874672085762432/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829159 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829134.857039:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7480 x1874672085762432/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829159 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829134.857049:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829134.857052:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829134.857056:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085762432:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829134.857063:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085762560:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829134.857086:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829134.857099:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829134/real 1787829134] req@ffff9e35da6f4780 x1874672085762560/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829159 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829134.857112:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4780 x1874672085762560/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829159 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829134.857121:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829134.857124:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829134.857128:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085762560:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829134.857134:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085762688:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829134.857154:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829134.857165:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829134/real 1787829134] req@ffff9e35da6f6d00 x1874672085762688/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829159 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829134.857176:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6d00 x1874672085762688/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829159 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829134.857184:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829134.857188:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829134.857191:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085762688:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829134.983924:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829134.983927:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829134.983966:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787829134.983989:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829136.007907:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829136.007908:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829136.007957:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829136.007963:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829137.031883:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829137.031887:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829137.031925:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787829137.031935:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829138.055886:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829138.055889:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829138.055934:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829138.055939:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829139.079922:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829139.079935:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829139.079961:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829139.080017:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829139.976025:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829139.976033:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787829139.976038:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829139.976057:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2575 00000100:00080000:0.0:1787829139.976069:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829139.976076:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787829139.976113:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7192c0 x1874672085762816/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829139.976159:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829139.976164:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829139.976167:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829139.976172:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2575 00000100:00080000:0.0:1787829139.976178:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829139.976183:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829139.976196:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829139.976202:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085762816:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1787829139.976222:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718780 x1874672085762944/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829139.976241:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829139.976245:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787829139.976264:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829139.976269:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2575 00000100:00080000:0.0:1787829139.976274:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829139.976279:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:2.0:1787829139.976284:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:0.0:1787829139.976286:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829139.976305:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a1c0 x1874672085763072/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829139.976317:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829139.976322:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:2.0:1787829139.976323:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829139/real 1787829139] req@ffff9e35da7192c0 x1874672085762816/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829164 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829139.976327:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787829139.976340:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7192c0 x1874672085762816/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829164 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829139.976346:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2575 00000100:00080000:2.0:1787829139.976354:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829139.976355:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829139.976360:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:0.0:1787829139.976365:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787829139.976366:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085762816:192.168.203.141@tcp:250:kworker.0 00000008:00100000:0.0:1787829139.976385:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:2.0:1787829139.976394:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085762944:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829139.976405:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7183c0 x1874672085763200/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829139.976415:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829139.976428:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829139/real 1787829139] req@ffff9e35da718780 x1874672085762944/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829164 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829139.976438:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787829139.976439:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718780 x1874672085762944/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829164 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829139.976449:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829139.976455:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829139.976456:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829139.976461:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829139.976467:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085762944:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787829139.976468:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2575 00000100:00100000:2.0:1787829139.976473:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085763072:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787829139.976477:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829139.976485:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:2.0:1787829139.976491:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:0.0:1787829139.976495:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:2.0:1787829139.976500:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829139/real 1787829139] req@ffff9e35da71a1c0 x1874672085763072/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829164 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829139.976510:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a1c0 x1874672085763072/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829164 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829139.976516:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a940 x1874672085763328/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829139.976518:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829139.976525:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829139.976531:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085763072:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787829139.976543:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829139.976548:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:2.0:1787829139.976552:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085763200:192.168.203.141@tcp:8:kworker.0 00000100:00080000:0.0:1787829139.976553:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829139.976563:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2575 00000100:00080000:0.0:1787829139.976569:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829139.976574:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829139.976581:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1787829139.976585:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829139.976605:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829139/real 1787829139] req@ffff9e35da7183c0 x1874672085763200/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829164 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829139.976610:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718f00 x1874672085763456/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829139.976638:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672085763200/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829164 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829139.976639:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829139.976650:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787829139.976654:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829139.976657:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829139.976663:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2575 00000100:00080000:2.0:1787829139.976665:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829139.976673:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085763200:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829139.976680:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085763328:192.168.203.141@tcp:8:kworker.0 00000100:00080000:0.0:1787829139.976684:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829139.976692:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829139.976699:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1787829139.976705:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829139.976716:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718000 x1874672085763584/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829139.976729:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829139/real 1787829139] req@ffff9e35da71a940 x1874672085763328/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829164 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829139.976731:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829139.976742:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829139.976746:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829139.976757:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2575 00000100:00100000:2.0:1787829139.976760:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672085763328/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829164 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829139.976763:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829139.976773:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:2.0:1787829139.976783:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000008:00100000:0.0:1787829139.976784:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00080000:2.0:1787829139.976790:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829139.976837:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085763328:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829139.976845:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a580 x1874672085763712/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829139.976859:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787829139.976859:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085763456:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787829139.976869:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829139.976873:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829139.976879:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2575 00000100:00080000:0.0:1787829139.976886:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829139.976891:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829139.976898:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000800:00000100:2.0:1787829139.976902:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829139.976915:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da719a40 x1874672085763840/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829139.976918:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829139/real 1787829139] req@ffff9e35da718f00 x1874672085763456/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829164 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829139.976943:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672085763456/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829164 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829139.976955:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829139.976958:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829139.976961:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085763456:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829139.976967:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085763584:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787829139.976985:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829139.976994:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829139/real 1787829139] req@ffff9e35da718000 x1874672085763584/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829164 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829139.977016:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718000 x1874672085763584/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829164 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829139.977021:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829139.977024:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829139.977026:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085763584:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829139.977042:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085763712:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829139.977055:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829139.977065:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829139/real 1787829139] req@ffff9e35da71a580 x1874672085763712/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829164 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829139.977084:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672085763712/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829164 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829139.977091:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829139.977097:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829139.977104:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085763712:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829139.977109:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085763840:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829139.977119:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829139.977126:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829139/real 1787829139] req@ffff9e35da719a40 x1874672085763840/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829164 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829139.977132:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672085763840/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829164 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829139.977137:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829139.977139:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829139.977141:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085763840:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829140.103938:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829140.103938:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829140.103938:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:1.0:1787829140.104002:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829141.127904:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829141.127905:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787829141.127927:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829141.128016:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829142.151898:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829142.151906:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787829142.151916:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829142.151998:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829143.175966:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829143.176002:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829143.176006:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787829143.176075:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829144.199902:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829144.199915:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829144.199927:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829144.199941:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829145.095883:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829145.095894:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787829145.095898:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829145.095909:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2580 00000100:00080000:0.0:1787829145.095920:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829145.095928:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787829145.095978:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718b40 x1874672085763968/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829145.096012:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829145.096015:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829145.096017:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829145.096020:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2580 00000100:00080000:0.0:1787829145.096024:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829145.096027:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829145.096038:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829145.096052:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da719a40 x1874672085764096/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829145.096062:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829145.096066:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:2.0:1787829145.096068:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085763968:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1787829145.096069:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829145.096077:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2580 00000100:00080000:0.0:1787829145.096081:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829145.096084:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829145.096087:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829145.096094:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a580 x1874672085764224/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829145.096101:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829145.096103:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829145.096104:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829145.096107:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2580 00000100:00080000:0.0:1787829145.096109:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829145.096112:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829145.096115:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:0.0:1787829145.096121:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718000 x1874672085764352/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829145.096127:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829145.096130:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829145.096132:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829145.096135:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2580 00000100:00080000:0.0:1787829145.096138:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829145.096140:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829145.096143:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:0.0:1787829145.096152:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718f00 x1874672085764480/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829145.096159:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1787829145.096161:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787829145.096162:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829145.096166:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829145.096171:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2580 00000100:00080000:0.0:1787829145.096174:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829145.096176:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829145.096179:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829145.096187:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a940 x1874672085764608/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829145.096194:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829145.096196:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787829145.096197:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829145.096199:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2580 00000100:00080000:0.0:1787829145.096201:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829145.096204:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829145.096207:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829145.096233:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7183c0 x1874672085764736/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829145.096238:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829145/real 1787829145] req@ffff9e35da718b40 x1874672085763968/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829170 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829145.096242:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829145.096247:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829145.096250:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829145.096259:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2580 00000100:00080000:0.0:1787829145.096265:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829145.096269:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829145.096273:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:2.0:1787829145.096280:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718b40 x1874672085763968/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829170 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829145.096282:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a1c0 x1874672085764864/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829145.096313:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829145.096318:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829145.096319:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829145.096323:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829145.096329:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:0.0:1787829145.096332:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2580 00000100:00080000:0.0:1787829145.096339:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787829145.096340:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085763968:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1787829145.096346:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829145.096356:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:0.0:1787829145.096367:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718780 x1874672085764992/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829145.096385:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085764096:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787829145.096428:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829145.096446:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829145/real 1787829145] req@ffff9e35da719a40 x1874672085764096/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829170 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829145.096462:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672085764096/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829170 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829145.096472:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829145.096477:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829145.096480:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085764096:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829145.096488:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085764224:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787829145.096511:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829145.096525:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829145/real 1787829145] req@ffff9e35da71a580 x1874672085764224/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829170 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829145.096538:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672085764224/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829170 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829145.096557:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829145.096560:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829145.096564:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085764224:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829145.096571:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085764352:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829145.096594:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829145.096607:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829145/real 1787829145] req@ffff9e35da718000 x1874672085764352/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829170 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829145.096621:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718000 x1874672085764352/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829170 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829145.096631:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829145.096635:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829145.096649:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085764352:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829145.096655:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085764480:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829145.096676:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829145.096687:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829145/real 1787829145] req@ffff9e35da718f00 x1874672085764480/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829170 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829145.096700:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672085764480/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829170 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829145.096710:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829145.096714:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829145.096718:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085764480:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829145.096724:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085764608:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787829145.096755:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829145.096767:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829145/real 1787829145] req@ffff9e35da71a940 x1874672085764608/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829170 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829145.096778:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672085764608/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829170 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829145.096793:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829145.096820:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829145.096826:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085764608:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829145.096833:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085764736:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787829145.096857:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829145.096870:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829145/real 1787829145] req@ffff9e35da7183c0 x1874672085764736/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829170 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829145.096881:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672085764736/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829170 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829145.096887:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829145.096890:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829145.096892:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085764736:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829145.096896:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085764864:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829145.096911:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829145.096918:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829145/real 1787829145] req@ffff9e35da71a1c0 x1874672085764864/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829170 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829145.096925:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a1c0 x1874672085764864/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829170 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829145.096932:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829145.096934:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829145.096936:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085764864:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829145.096941:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085764992:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829145.096951:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829145.096957:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829145/real 1787829145] req@ffff9e35da718780 x1874672085764992/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829170 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829145.096964:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718780 x1874672085764992/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829170 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829145.096970:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829145.096972:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829145.096974:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085764992:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829145.223912:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829145.223912:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829145.223925:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787829145.223967:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829146.247904:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829146.247915:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829146.247915:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:1.0:1787829146.248009:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829147.271927:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829147.271927:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829147.271927:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829147.271995:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829148.295874:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829148.295888:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829148.295909:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787829148.295971:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829149.319943:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829149.319956:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829149.319978:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787829149.319998:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829150.215986:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829150.215997:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787829150.216002:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829150.216013:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2585 00000100:00080000:3.0:1787829150.216026:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829150.216033:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787829150.216077:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499a40 x1874672085765120/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829150.216107:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829150.216110:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829150.216111:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829150.216114:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2585 00000100:00080000:3.0:1787829150.216117:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829150.216119:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829150.216129:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829150.216139:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498780 x1874672085765248/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829150.216147:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829150.216150:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787829150.216151:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829150.216154:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2585 00000100:00080000:3.0:1787829150.216156:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787829150.216161:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085765120:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1787829150.216161:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829150.216168:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829150.216194:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a940 x1874672085765376/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829150.216202:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829150.216205:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787829150.216207:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829150.216210:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2585 00000100:00080000:3.0:1787829150.216214:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829150.216217:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829150.216221:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:3.0:1787829150.216231:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498b40 x1874672085765504/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829150.216238:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829150.216240:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787829150.216241:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829150.216244:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2585 00000100:00080000:3.0:1787829150.216247:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1787829150.216252:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1787829150.216253:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829150.216259:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:3.0:1787829150.216270:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649bc00 x1874672085765632/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829150.216276:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829150.216279:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829150.216280:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829150.216283:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2585 00000100:00080000:3.0:1787829150.216286:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829150.216288:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829150.216293:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829150.216301:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829150/real 1787829150] req@ffff9e35c6499a40 x1874672085765120/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829175 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829150.216304:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b840 x1874672085765760/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829150.216332:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829150.216334:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787829150.216336:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787829150.216341:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499a40 x1874672085765120/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829175 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829150.216342:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2585 00000100:00080000:3.0:1787829150.216347:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829150.216353:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829150.216360:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1787829150.216374:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1787829150.216378:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498f00 x1874672085765888/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829150.216381:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829150.216389:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085765120:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1787829150.216392:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829150.216396:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787829150.216398:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787829150.216403:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085765248:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1787829150.216405:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2585 00000100:00080000:3.0:1787829150.216409:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829150.216413:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829150.216417:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:3.0:1787829150.216425:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499680 x1874672085766016/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829150.216432:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829150.216435:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787829150.216437:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1787829150.216441:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1787829150.216444:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2585 00000100:00080000:3.0:1787829150.216449:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829150.216451:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829150.216455:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:2.0:1787829150.216463:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829150/real 1787829150] req@ffff9e35c6498780 x1874672085765248/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829175 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829150.216466:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a1c0 x1874672085766144/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829150.216499:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498780 x1874672085765248/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829175 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829150.216507:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829150.216510:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829150.216512:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085765248:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829150.216544:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085765376:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787829150.216568:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829150.216577:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829150/real 1787829150] req@ffff9e35c649a940 x1874672085765376/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829175 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829150.216586:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a940 x1874672085765376/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829175 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829150.216592:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829150.216595:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829150.216597:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085765376:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829150.216602:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085765504:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829150.216614:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829150.216621:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829150/real 1787829150] req@ffff9e35c6498b40 x1874672085765504/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829175 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829150.216629:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498b40 x1874672085765504/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829175 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829150.216635:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829150.216637:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829150.216638:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085765504:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829150.216642:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085765632:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829150.216652:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829150.216658:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829150/real 1787829150] req@ffff9e35c649bc00 x1874672085765632/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829175 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829150.216666:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649bc00 x1874672085765632/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829175 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829150.216671:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829150.216673:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829150.216675:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085765632:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829150.216679:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085765760:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787829150.216689:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829150.216695:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829150/real 1787829150] req@ffff9e35c649b840 x1874672085765760/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829175 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829150.216703:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b840 x1874672085765760/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829175 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829150.216709:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829150.216711:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829150.216712:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085765760:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829150.216716:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085765888:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787829150.216726:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829150.216732:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829150/real 1787829150] req@ffff9e35c6498f00 x1874672085765888/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829175 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829150.216740:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498f00 x1874672085765888/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829175 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829150.216745:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829150.216747:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829150.216749:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085765888:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829150.216752:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085766016:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829150.216762:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829150.216767:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829150/real 1787829150] req@ffff9e35c6499680 x1874672085766016/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829175 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829150.216774:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499680 x1874672085766016/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829175 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829150.216785:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829150.216787:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829150.216789:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085766016:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829150.216792:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085766144:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829150.216836:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829150.216849:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829150/real 1787829150] req@ffff9e35c649a1c0 x1874672085766144/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829175 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829150.216861:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a1c0 x1874672085766144/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829175 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829150.216868:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829150.216870:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829150.216873:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085766144:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829150.343866:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829150.343881:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829150.343931:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829150.343944:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829151.367940:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829151.367940:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829151.368002:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:1.0:1787829151.368105:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829152.392003:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829152.392004:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829152.392074:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787829152.392115:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829153.415925:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:3.0:1787829153.415935:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829153.415943:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829153.416009:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829154.439925:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:3.0:1787829154.439925:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829154.439948:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787829154.440016:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829155.335922:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829155.335933:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787829155.335937:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829155.335949:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2590 00000100:00080000:0.0:1787829155.335961:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829155.335969:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787829155.336012:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7192c0 x1874672085766272/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829155.336040:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829155.336043:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829155.336045:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829155.336048:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2590 00000100:00080000:0.0:1787829155.336050:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829155.336053:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829155.336063:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829155.336075:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718780 x1874672085766400/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829155.336082:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829155.336085:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787829155.336087:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829155.336089:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2590 00000100:00080000:0.0:1787829155.336091:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829155.336096:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787829155.336098:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085766272:192.168.203.141@tcp:250:kworker.0 00000008:00100000:0.0:1787829155.336100:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829155.336111:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a1c0 x1874672085766528/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829155.336119:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829155.336121:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829155.336123:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829155.336126:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2590 00000100:00080000:0.0:1787829155.336128:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829155.336130:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829155.336134:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:0.0:1787829155.336156:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7183c0 x1874672085766656/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829155.336163:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829155.336167:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829155.336168:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829155.336172:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2590 00000100:00080000:0.0:1787829155.336175:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829155.336178:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829155.336181:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:0.0:1787829155.336192:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a940 x1874672085766784/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829155.336193:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787829155.336202:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829155.336206:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829155.336207:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829155.336211:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2590 00000100:00080000:0.0:1787829155.336214:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829155.336217:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829155.336220:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829155.336228:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718f00 x1874672085766912/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829155.336235:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829155.336237:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787829155.336238:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829155.336241:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2590 00000100:00080000:0.0:1787829155.336245:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787829155.336248:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829155/real 1787829155] req@ffff9e35da7192c0 x1874672085766272/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829180 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829155.336251:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829155.336258:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829155.336269:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718000 x1874672085767040/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829155.336273:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7192c0 x1874672085766272/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829180 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829155.336287:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829155.336291:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829155.336293:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829155.336293:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829155.336298:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2590 00000100:00080000:2.0:1787829155.336300:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:0.0:1787829155.336302:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787829155.336306:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085766272:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1787829155.336310:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829155.336316:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:2.0:1787829155.336318:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085766400:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829155.336325:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a580 x1874672085767168/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829155.336338:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1787829155.336341:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787829155.336343:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829155.336345:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829155.336350:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2590 00000100:00080000:0.0:1787829155.336353:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787829155.336356:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829155/real 1787829155] req@ffff9e35da718780 x1874672085766400/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829180 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829155.336357:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829155.336364:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:0.0:1787829155.336376:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da719a40 x1874672085767296/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829155.336378:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718780 x1874672085766400/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829180 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829155.336402:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829155.336405:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829155.336408:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085766400:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829155.336433:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085766528:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787829155.336445:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829155.336452:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829155/real 1787829155] req@ffff9e35da71a1c0 x1874672085766528/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829180 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829155.336460:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a1c0 x1874672085766528/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829180 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829155.336471:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829155.336473:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829155.336475:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085766528:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829155.336478:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085766656:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829155.336488:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829155.336494:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829155/real 1787829155] req@ffff9e35da7183c0 x1874672085766656/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829180 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829155.336501:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672085766656/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829180 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829155.336507:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829155.336509:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829155.336511:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085766656:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829155.336515:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085766784:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829155.336525:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829155.336531:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829155/real 1787829155] req@ffff9e35da71a940 x1874672085766784/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829180 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829155.336539:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672085766784/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829180 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829155.336544:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829155.336546:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829155.336548:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085766784:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829155.336552:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085766912:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787829155.336561:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829155.336567:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829155/real 1787829155] req@ffff9e35da718f00 x1874672085766912/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829180 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829155.336574:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672085766912/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829180 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829155.336579:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829155.336580:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829155.336582:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085766912:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829155.336586:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085767040:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787829155.336595:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829155.336601:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829155/real 1787829155] req@ffff9e35da718000 x1874672085767040/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829180 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829155.336608:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718000 x1874672085767040/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829180 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829155.336613:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829155.336615:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829155.336617:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085767040:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829155.336620:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085767168:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829155.336631:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829155.336637:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829155/real 1787829155] req@ffff9e35da71a580 x1874672085767168/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829180 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829155.336644:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672085767168/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829180 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829155.336650:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829155.336652:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829155.336653:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085767168:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829155.336657:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085767296:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829155.336666:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829155.336671:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829155/real 1787829155] req@ffff9e35da719a40 x1874672085767296/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829180 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829155.336678:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672085767296/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829180 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829155.336684:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829155.336686:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829155.336688:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085767296:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787829155.463924:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829155.463925:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829155.463925:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787829155.464057:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829156.487881:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:3.0:1787829156.487888:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829156.487898:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829156.487965:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829157.511880:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829157.511880:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829157.511948:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787829157.511963:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829158.535901:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829158.535969:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:3.0:1787829158.535983:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787829158.535989:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829159.559885:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:3.0:1787829159.559885:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829159.559942:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829159.559950:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829160.455934:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829160.455945:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829160.455951:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829160.455963:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2595 00000100:00080000:1.0:1787829160.455974:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829160.455982:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787829160.456022:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6940 x1874672085767424/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829160.456052:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829160.456057:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829160.456059:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829160.456064:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2595 00000100:00080000:1.0:1787829160.456069:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829160.456075:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829160.456086:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829160.456091:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085767424:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1787829160.456121:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6580 x1874672085767552/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829160.456136:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829160.456140:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829160.456143:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829160.456148:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2595 00000100:00080000:1.0:1787829160.456154:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829160.456159:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:2.0:1787829160.456164:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:1.0:1787829160.456168:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829160.456186:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f43c0 x1874672085767680/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829160.456198:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787829160.456202:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829160/real 1787829160] req@ffff9e35da6f6940 x1874672085767424/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829185 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829160.456204:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829160.456208:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829160.456219:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2595 00000100:00080000:1.0:1787829160.456225:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787829160.456226:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6940 x1874672085767424/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829185 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829160.456235:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:2.0:1787829160.456238:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:1.0:1787829160.456243:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00080000:2.0:1787829160.456244:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829160.456249:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085767424:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1787829160.456263:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f61c0 x1874672085767808/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829160.456275:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085767552:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787829160.456278:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829160.456283:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829160.456286:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829160.456292:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2595 00000800:00000100:2.0:1787829160.456297:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787829160.456300:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829160.456307:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787829160.456313:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829160/real 1787829160] req@ffff9e35da6f6580 x1874672085767552/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829185 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1787829160.456314:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:1.0:1787829160.456333:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4f00 x1874672085767936/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829160.456335:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6580 x1874672085767552/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829185 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829160.456359:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829160.456361:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829160.456366:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829160.456368:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:1.0:1787829160.456371:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787829160.456373:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085767552:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787829160.456379:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2595 00000100:00100000:2.0:1787829160.456382:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085767680:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787829160.456385:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829160.456393:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829160.456399:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1787829160.456404:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829160.456415:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6940 x1874672085768064/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829160.456420:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829160/real 1787829160] req@ffff9e35da6f43c0 x1874672085767680/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829185 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829160.456435:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829160.456443:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829160.456446:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829160.456456:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2595 00000100:00080000:1.0:1787829160.456462:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787829160.456467:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f43c0 x1874672085767680/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829185 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829160.456470:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829160.456480:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1787829160.456481:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829160.456487:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829160.456491:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085767680:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829160.456498:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6d00 x1874672085768192/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829160.456507:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085767808:192.168.203.141@tcp:8:kworker.0 00000100:00080000:1.0:1787829160.456521:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1787829160.456527:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787829160.456529:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829160.456533:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787829160.456540:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829160/real 1787829160] req@ffff9e35da6f61c0 x1874672085767808/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829185 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829160.456541:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2595 00000100:00080000:1.0:1787829160.456549:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787829160.456556:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f61c0 x1874672085767808/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829185 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829160.456558:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829160.456568:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00080000:2.0:1787829160.456569:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829160.456575:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829160.456577:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085767808:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829160.456582:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085767936:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829160.456587:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4780 x1874672085768320/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829160.456598:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787829160.456599:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829160.456605:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:2.0:1787829160.456607:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829160/real 1787829160] req@ffff9e35da6f4f00 x1874672085767936/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829185 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829160.456609:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829160.456619:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2595 00000100:00100000:2.0:1787829160.456623:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4f00 x1874672085767936/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829185 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829160.456625:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829160.456634:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829160.456643:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00080000:2.0:1787829160.456644:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829160.456650:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829160.456653:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085767936:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829160.456661:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7480 x1874672085768448/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829160.456661:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085768064:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787829160.456682:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829160.456691:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829160/real 1787829160] req@ffff9e35da6f6940 x1874672085768064/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829185 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829160.456698:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6940 x1874672085768064/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829185 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829160.456704:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829160.456706:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829160.456708:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085768064:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829160.456720:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085768192:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787829160.456729:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829160.456735:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829160/real 1787829160] req@ffff9e35da6f6d00 x1874672085768192/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829185 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829160.456742:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6d00 x1874672085768192/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829185 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829160.456748:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829160.456749:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829160.456761:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085768192:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829160.456765:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085768320:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829160.456776:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829160.456782:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829160/real 1787829160] req@ffff9e35da6f4780 x1874672085768320/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829185 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829160.456790:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4780 x1874672085768320/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829185 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829160.456795:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829160.456824:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829160.456827:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085768320:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829160.456831:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085768448:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829160.456849:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829160.456857:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829160/real 1787829160] req@ffff9e35da6f7480 x1874672085768448/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829185 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829160.456865:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7480 x1874672085768448/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829185 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829160.456871:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829160.456872:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829160.456874:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085768448:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829160.583877:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829160.583927:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829160.583935:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829160.583950:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829161.607872:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829161.607883:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829161.607912:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829161.607922:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829162.631886:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829162.631912:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829162.631954:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829162.631984:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829163.655870:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829163.655900:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829163.655919:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829163.655923:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829164.679889:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829164.679889:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829164.679920:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829164.679930:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829165.575929:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829165.575943:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787829165.575948:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829165.575962:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2600 00000100:00080000:0.0:1787829165.575974:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829165.575982:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787829165.576026:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718b40 x1874672085768576/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829165.576059:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829165.576061:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829165.576062:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829165.576065:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2600 00000100:00080000:0.0:1787829165.576067:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829165.576084:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829165.576095:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829165.576109:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da719a40 x1874672085768704/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829165.576115:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085768576:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1787829165.576121:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829165.576125:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787829165.576127:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829165.576130:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2600 00000100:00080000:0.0:1787829165.576133:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829165.576135:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829165.576139:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829165.576147:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a580 x1874672085768832/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829165.576154:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829165.576157:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829165.576158:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829165.576161:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2600 00000100:00080000:0.0:1787829165.576165:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829165.576167:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829165.576171:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:0.0:1787829165.576178:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718000 x1874672085768960/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829165.576184:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829165.576189:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000800:00000100:2.0:1787829165.576190:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787829165.576192:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829165.576197:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2600 00000100:00080000:0.0:1787829165.576200:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829165.576203:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829165.576206:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:0.0:1787829165.576214:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718f00 x1874672085769088/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829165.576221:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829165.576224:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829165.576225:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829165.576229:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2600 00000100:00080000:0.0:1787829165.576232:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829165.576235:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787829165.576239:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829165/real 1787829165] req@ffff9e35da718b40 x1874672085768576/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829190 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1787829165.576241:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829165.576256:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a940 x1874672085769216/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829165.576263:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718b40 x1874672085768576/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829190 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829165.576284:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829165.576287:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787829165.576288:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829165.576294:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2600 00000100:00080000:2.0:1787829165.576296:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829165.576298:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829165.576302:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:0.0:1787829165.576307:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787829165.576311:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085768576:192.168.203.141@tcp:250:kworker.0 00000008:00100000:0.0:1787829165.576315:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829165.576331:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7183c0 x1874672085769344/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829165.576339:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085768704:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787829165.576351:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829165.576355:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829165.576357:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829165.576362:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2600 00000800:00000100:2.0:1787829165.576366:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787829165.576367:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829165.576374:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829165.576379:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:2.0:1787829165.576382:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829165/real 1787829165] req@ffff9e35da719a40 x1874672085768704/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829190 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829165.576391:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a1c0 x1874672085769472/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829165.576398:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672085768704/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829190 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829165.576405:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829165.576412:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829165.576415:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829165.576420:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829165.576421:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2600 00000100:00080000:0.0:1787829165.576428:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829165.576430:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:0.0:1787829165.576434:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787829165.576437:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085768704:192.168.203.141@tcp:38:kworker.0 00000008:00100000:0.0:1787829165.576439:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:2.0:1787829165.576449:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085768832:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829165.576454:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718780 x1874672085769600/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829165.576476:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829165.576487:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829165/real 1787829165] req@ffff9e35da71a580 x1874672085768832/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829190 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829165.576494:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672085768832/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829190 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829165.576500:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829165.576502:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829165.576504:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085768832:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829165.576508:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085768960:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829165.576527:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829165.576538:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829165/real 1787829165] req@ffff9e35da718000 x1874672085768960/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829190 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829165.576552:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718000 x1874672085768960/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829190 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829165.576562:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829165.576565:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829165.576568:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085768960:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829165.576573:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085769088:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829165.576595:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829165.576604:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829165/real 1787829165] req@ffff9e35da718f00 x1874672085769088/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829190 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829165.576618:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672085769088/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829190 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829165.576624:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829165.576626:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829165.576628:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085769088:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829165.576632:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085769216:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787829165.576644:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829165.576657:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829165/real 1787829165] req@ffff9e35da71a940 x1874672085769216/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829190 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829165.576665:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672085769216/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829190 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829165.576671:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829165.576673:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829165.576675:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085769216:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829165.576690:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085769344:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787829165.576701:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829165.576708:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829165/real 1787829165] req@ffff9e35da7183c0 x1874672085769344/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829190 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829165.576715:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672085769344/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829190 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829165.576725:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829165.576727:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829165.576729:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085769344:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829165.576733:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085769472:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829165.576742:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829165.576748:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829165/real 1787829165] req@ffff9e35da71a1c0 x1874672085769472/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829190 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829165.576755:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a1c0 x1874672085769472/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829190 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829165.576761:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829165.576762:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829165.576764:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085769472:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829165.576768:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085769600:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829165.576777:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829165.576783:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829165/real 1787829165] req@ffff9e35da718780 x1874672085769600/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829190 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829165.576790:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718780 x1874672085769600/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829190 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829165.576796:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829165.576821:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829165.576824:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085769600:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829165.703888:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787829165.703897:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829165.703916:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829165.703972:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829166.727956:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829166.727956:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829166.728121:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:1.0:1787829166.728158:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829167.751916:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829167.751991:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787829167.751993:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829167.752001:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829168.775943:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829168.775970:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829168.776003:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829168.776168:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829169.799919:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829169.799930:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829169.799939:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:1.0:1787829169.799970:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829170.695937:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829170.695946:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787829170.695950:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829170.695959:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2606 00000100:00080000:3.0:1787829170.695969:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829170.695976:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787829170.696012:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a580 x1874672085769728/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829170.696060:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829170.696067:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829170.696068:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829170.696072:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2606 00000100:00080000:3.0:1787829170.696075:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829170.696079:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829170.696089:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829170.696100:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b0c0 x1874672085769856/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829170.696110:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829170.696114:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787829170.696116:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829170.696120:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2606 00000100:00080000:3.0:1787829170.696122:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829170.696125:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829170.696128:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829170.696137:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b480 x1874672085769984/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829170.696146:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829170.696151:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787829170.696159:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787829170.696161:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085769728:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1787829170.696168:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2606 00000100:00080000:3.0:1787829170.696180:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829170.696185:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829170.696189:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:3.0:1787829170.696221:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c64992c0 x1874672085770112/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829170.696228:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829170.696230:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787829170.696232:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829170.696235:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2606 00000100:00080000:3.0:1787829170.696238:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829170.696241:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829170.696244:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:3.0:1787829170.696253:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499e00 x1874672085770240/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829170.696259:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829170.696263:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829170.696265:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829170.696267:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2606 00000100:00080000:3.0:1787829170.696270:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829170.696272:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829170.696277:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1787829170.696280:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787829170.696294:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498000 x1874672085770368/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829170.696302:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829170.696305:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787829170.696307:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829170.696310:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2606 00000100:00080000:3.0:1787829170.696313:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829170.696317:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829170.696320:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829170.696327:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649ad00 x1874672085770496/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829170.696336:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829170.696338:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787829170.696339:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829170.696342:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2606 00000100:00080000:3.0:1787829170.696346:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829170.696348:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829170.696352:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:2.0:1787829170.696361:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829170/real 1787829170] req@ffff9e35c649a580 x1874672085769728/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829195 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829170.696361:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c64983c0 x1874672085770624/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829170.696421:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829170.696434:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787829170.696436:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829170.696441:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2606 00000100:00080000:3.0:1787829170.696444:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787829170.696453:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a580 x1874672085769728/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829195 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829170.696454:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829170.696465:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00080000:2.0:1787829170.696491:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829170.696498:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787829170.696505:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a1c0 x1874672085770752/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829170.696507:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085769728:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1787829170.696523:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085769856:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787829170.696566:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829170.696586:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829170/real 1787829170] req@ffff9e35c649b0c0 x1874672085769856/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829195 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829170.696597:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b0c0 x1874672085769856/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829195 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829170.696606:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829170.696610:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829170.696614:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085769856:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829170.696653:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085769984:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787829170.696689:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829170.696705:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829170/real 1787829170] req@ffff9e35c649b480 x1874672085769984/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829195 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829170.696722:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b480 x1874672085769984/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829195 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829170.696734:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829170.696739:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829170.696743:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085769984:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829170.696762:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c64992c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085770112:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829170.696783:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829170.696794:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829170/real 1787829170] req@ffff9e35c64992c0 x1874672085770112/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829195 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829170.696860:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c64992c0 x1874672085770112/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829195 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829170.696872:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829170.696877:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829170.696882:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c64992c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085770112:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829170.696891:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085770240:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829170.696916:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829170.696927:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829170/real 1787829170] req@ffff9e35c6499e00 x1874672085770240/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829195 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829170.696937:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499e00 x1874672085770240/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829195 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829170.696945:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829170.696948:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829170.696950:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085770240:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829170.696956:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085770368:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787829170.696970:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829170.696977:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829170/real 1787829170] req@ffff9e35c6498000 x1874672085770368/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829195 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829170.696986:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498000 x1874672085770368/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829195 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829170.696993:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829170.696995:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829170.696998:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085770368:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829170.697003:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085770496:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787829170.697015:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829170.697022:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829170/real 1787829170] req@ffff9e35c649ad00 x1874672085770496/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829195 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829170.697058:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649ad00 x1874672085770496/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829195 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829170.697066:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829170.697068:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829170.697071:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085770496:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829170.697076:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c64983c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085770624:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829170.697095:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829170.697104:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829170/real 1787829170] req@ffff9e35c64983c0 x1874672085770624/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829195 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829170.697113:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c64983c0 x1874672085770624/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829195 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829170.697120:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829170.697123:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829170.697126:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c64983c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085770624:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829170.697131:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085770752:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829170.697143:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829170.697149:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829170/real 1787829170] req@ffff9e35c649a1c0 x1874672085770752/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829195 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829170.697159:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a1c0 x1874672085770752/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829195 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829170.697171:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829170.697173:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829170.697176:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085770752:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829170.823970:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829170.823979:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829170.824070:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787829170.824195:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829171.847920:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829171.847923:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829171.847962:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829171.847993:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829172.871935:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829172.871935:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829172.871968:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787829172.872007:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829173.895954:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829173.895954:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829173.895966:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787829173.896027:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829174.919946:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829174.919985:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829174.920022:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829174.920026:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829175.815974:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829175.815984:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829175.815989:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829175.816015:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2611 00000100:00080000:2.0:1787829175.816029:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829175.816038:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787829175.816081:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c3c0 x1874672085770880/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829175.816111:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829175.816115:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829175.816117:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829175.816125:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2611 00000100:00080000:2.0:1787829175.816128:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829175.816131:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829175.816141:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829175.816153:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085770880:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1787829175.816157:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498ed00 x1874672085771008/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829175.816187:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829175.816190:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829175.816193:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829175.816200:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2611 00000100:00080000:2.0:1787829175.816205:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829175.816209:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829175.816213:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829175.816243:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498de00 x1874672085771136/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829175.816253:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1787829175.816263:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1787829175.816265:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829175.816269:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829175.816276:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2611 00000100:00080000:2.0:1787829175.816298:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829175.816302:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829175.816309:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:2.0:1787829175.816318:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498fc00 x1874672085771264/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829175.816325:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829175.816332:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:0.0:1787829175.816333:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829175/real 1787829175] req@ffff9e35c498c3c0 x1874672085770880/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829200 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829175.816335:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829175.816363:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2611 00000100:00080000:2.0:1787829175.816368:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829175.816374:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787829175.816375:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c3c0 x1874672085770880/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829200 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1787829175.816384:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00080000:0.0:1787829175.816404:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829175.816415:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829175.816415:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e940 x1874672085771392/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829175.816431:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085770880:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1787829175.816443:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829175.816448:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829175.816450:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829175.816455:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2611 00000100:00080000:2.0:1787829175.816459:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829175.816463:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787829175.816486:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085771008:192.168.203.141@tcp:38:kworker.0 00000008:00100000:2.0:1787829175.816486:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829175.816519:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c780 x1874672085771520/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829175.816528:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1787829175.816529:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829175.816533:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829175.816536:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829175.816543:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2611 00000100:00080000:2.0:1787829175.816546:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787829175.816547:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829175/real 1787829175] req@ffff9e35c498ed00 x1874672085771008/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829200 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829175.816554:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829175.816562:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829175.816570:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498ed00 x1874672085771008/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829200 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829175.816576:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498d680 x1874672085771648/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829175.816623:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829175.816628:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829175.816634:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829175.816635:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:2.0:1787829175.816637:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829175.816648:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2611 00000100:00080000:2.0:1787829175.816665:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787829175.816669:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085771008:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1787829175.816672:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829175.816680:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:0.0:1787829175.816691:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085771136:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829175.816692:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c3c0 x1874672085771776/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829175.816716:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829175.816720:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829175.816722:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829175.816725:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2611 00000100:00080000:2.0:1787829175.816729:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829175.816732:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829175.816738:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:2.0:1787829175.816746:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cf00 x1874672085771904/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829175.816761:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829175.816786:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829175/real 1787829175] req@ffff9e35c498de00 x1874672085771136/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829200 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829175.816821:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498de00 x1874672085771136/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829200 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829175.816830:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829175.816833:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829175.816837:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085771136:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829175.816843:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085771264:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829175.816878:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829175.816887:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829175/real 1787829175] req@ffff9e35c498fc00 x1874672085771264/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829200 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829175.816901:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498fc00 x1874672085771264/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829200 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829175.816908:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829175.816910:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829175.816912:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085771264:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829175.816917:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085771392:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829175.816929:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829175.816935:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829175/real 1787829175] req@ffff9e35c498e940 x1874672085771392/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829200 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829175.816956:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e940 x1874672085771392/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829200 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829175.816963:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829175.816965:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829175.816968:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085771392:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829175.817016:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085771520:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829175.817032:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829175.817039:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829175/real 1787829175] req@ffff9e35c498c780 x1874672085771520/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829200 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829175.817047:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c780 x1874672085771520/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829200 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829175.817058:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829175.817060:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829175.817062:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085771520:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829175.817066:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085771648:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829175.817076:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829175.817091:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829175/real 1787829175] req@ffff9e35c498d680 x1874672085771648/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829200 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829175.817098:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498d680 x1874672085771648/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829200 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829175.817103:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829175.817105:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829175.817107:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085771648:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829175.817111:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085771776:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829175.817120:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829175.817126:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829175/real 1787829175] req@ffff9e35c498c3c0 x1874672085771776/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829200 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829175.817133:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c3c0 x1874672085771776/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829200 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829175.817138:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829175.817140:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829175.817142:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085771776:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829175.817146:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085771904:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829175.817162:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829175.817172:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829175/real 1787829175] req@ffff9e35c498cf00 x1874672085771904/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829200 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829175.817184:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498cf00 x1874672085771904/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829200 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829175.817193:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829175.817196:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829175.817199:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085771904:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829175.943917:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829175.943922:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787829175.943982:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829175.944051:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829176.967895:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:0.0:1787829176.967957:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829176.967965:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829176.968028:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829177.991926:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829177.991926:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:3.0:1787829177.991926:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829177.992022:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829179.015946:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:3.0:1787829179.015946:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829179.015972:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:1.0:1787829179.016188:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829180.039913:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829180.039925:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829180.039960:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787829180.040041:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829180.935927:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829180.935942:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829180.935947:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829180.935973:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2616 00000100:00080000:1.0:1787829180.935988:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829180.935998:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787829180.936045:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4000 x1874672085772032/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829180.936079:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829180.936084:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829180.936086:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829180.936091:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2616 00000100:00080000:1.0:1787829180.936095:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829180.936100:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829180.936113:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829180.936129:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5e00 x1874672085772160/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829180.936133:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085772032:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787829180.936151:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829180.936155:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829180.936157:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829180.936164:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2616 00000100:00080000:1.0:1787829180.936168:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829180.936173:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829180.936179:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829180.936197:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4b40 x1874672085772288/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829180.936208:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829180.936212:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829180.936214:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829180.936219:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2616 00000100:00080000:1.0:1787829180.936223:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829180.936227:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829180.936232:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:1.0:1787829180.936243:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f70c0 x1874672085772416/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829180.936254:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787829180.936257:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829180.936262:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829180.936264:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829180.936269:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2616 00000100:00080000:1.0:1787829180.936274:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829180.936278:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829180.936282:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:1.0:1787829180.936296:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7840 x1874672085772544/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829180.936306:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787829180.936312:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829180/real 1787829180] req@ffff9e35da6f4000 x1874672085772032/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829205 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829180.936312:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829180.936317:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829180.936327:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2616 00000100:00080000:1.0:1787829180.936332:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787829180.936338:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4000 x1874672085772032/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829205 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829180.936339:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829180.936350:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1787829180.936354:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829180.936360:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829180.936367:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085772032:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1787829180.936368:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5a40 x1874672085772672/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829180.936384:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829180.936388:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829180.936390:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829180.936395:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2616 00000100:00100000:0.0:1787829180.936400:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085772160:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787829180.936401:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829180.936408:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829180.936414:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829180.936427:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4000 x1874672085772800/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829180.936438:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787829180.936452:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787829180.936455:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829180/real 1787829180] req@ffff9e35da6f5e00 x1874672085772160/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829205 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829180.936456:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829180.936460:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829180.936469:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2616 00000100:00080000:1.0:1787829180.936473:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787829180.936476:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5e00 x1874672085772160/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829205 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829180.936481:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829180.936491:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00080000:0.0:1787829180.936493:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829180.936499:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829180.936502:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085772160:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829180.936506:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7480 x1874672085772928/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829180.936514:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829180.936516:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829180.936517:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787829180.936520:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085772288:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787829180.936523:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2616 00000100:00080000:1.0:1787829180.936527:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829180.936531:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829180.936535:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:1.0:1787829180.936547:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4780 x1874672085773056/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829180.936552:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829180.936566:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829180/real 1787829180] req@ffff9e35da6f4b40 x1874672085772288/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829205 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829180.936575:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4b40 x1874672085772288/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829205 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829180.936583:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829180.936586:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829180.936588:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085772288:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829180.936593:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085772416:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829180.936605:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829180.936612:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829180/real 1787829180] req@ffff9e35da6f70c0 x1874672085772416/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829205 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829180.936620:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f70c0 x1874672085772416/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829205 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829180.936626:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829180.936627:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829180.936629:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085772416:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829180.936633:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085772544:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829180.936643:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829180.936649:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829180/real 1787829180] req@ffff9e35da6f7840 x1874672085772544/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829205 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829180.936659:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7840 x1874672085772544/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829205 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829180.936668:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829180.936671:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829180.936675:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085772544:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829180.936681:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085772672:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829180.936702:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829180.936728:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829180/real 1787829180] req@ffff9e35da6f5a40 x1874672085772672/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829205 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829180.936737:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5a40 x1874672085772672/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829205 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829180.936748:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829180.936751:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829180.936753:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085772672:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829180.936772:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085772800:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829180.936786:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829180.936794:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829180/real 1787829180] req@ffff9e35da6f4000 x1874672085772800/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829205 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829180.936823:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4000 x1874672085772800/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829205 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829180.936829:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829180.936831:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829180.936833:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085772800:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829180.936837:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085772928:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829180.936850:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829180.936857:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829180/real 1787829180] req@ffff9e35da6f7480 x1874672085772928/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829205 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829180.936865:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7480 x1874672085772928/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829205 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829180.936871:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829180.936873:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829180.936876:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085772928:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829180.936879:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085773056:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829180.936889:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829180.936895:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829180/real 1787829180] req@ffff9e35da6f4780 x1874672085773056/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829205 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829180.936902:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4780 x1874672085773056/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829205 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829180.936907:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829180.936909:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829180.936911:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085773056:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829181.063884:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829181.063884:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829181.063915:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:0.0:1787829181.063936:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829182.087977:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829182.087977:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829182.087994:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829182.088045:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829183.111932:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829183.111932:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829183.111967:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787829183.112030:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829184.135951:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829184.135954:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829184.135969:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:0.0:1787829184.136021:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829185.159910:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829185.159947:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829185.159951:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787829185.159993:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829186.055886:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829186.055898:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829186.055903:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829186.055915:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2621 00000100:00080000:1.0:1787829186.055961:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829186.055982:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787829186.056035:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6d00 x1874672085773184/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829186.056060:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829186.056064:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829186.056066:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829186.056072:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2621 00000100:00080000:1.0:1787829186.056074:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829186.056078:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829186.056090:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829186.056110:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6940 x1874672085773312/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829186.056125:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829186.056133:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:0.0:1787829186.056135:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085773184:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787829186.056136:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829186.056156:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2621 00000100:00080000:1.0:1787829186.056160:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829186.056163:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829186.056177:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829186.056191:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4f00 x1874672085773440/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829186.056197:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829186.056200:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829186.056202:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829186.056207:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2621 00000100:00080000:1.0:1787829186.056209:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829186.056211:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829186.056214:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:1.0:1787829186.056224:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f61c0 x1874672085773568/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829186.056231:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829186.056235:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829186.056236:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829186.056239:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2621 00000100:00080000:1.0:1787829186.056243:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829186.056245:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829186.056249:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:1.0:1787829186.056261:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f43c0 x1874672085773696/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829186.056270:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829186.056274:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829186.056276:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829186.056281:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2621 00000100:00080000:1.0:1787829186.056284:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829186.056287:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829186.056291:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829186.056308:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6580 x1874672085773824/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829186.056311:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085773312:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787829186.056320:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829186.056328:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829186.056330:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829186.056334:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2621 00000100:00080000:1.0:1787829186.056340:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829186.056343:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829186.056346:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829186.056358:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4780 x1874672085773952/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829186.056365:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829186.056369:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829186.056370:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829186.056374:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2621 00000100:00080000:1.0:1787829186.056379:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787829186.056384:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085773440:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787829186.056393:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829186.056405:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:0.0:1787829186.056424:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085773568:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829186.056430:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7480 x1874672085774080/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829186.056447:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829186.056451:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829186.056454:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829186.056460:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2621 00000100:00080000:1.0:1787829186.056467:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787829186.056475:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085773696:192.168.203.141@tcp:8:kworker.0 00000100:00080000:1.0:1787829186.056476:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829186.056489:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:1.0:1787829186.056510:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4000 x1874672085774208/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829186.056540:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085773824:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829186.056595:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085773952:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829186.056624:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085774080:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829186.056651:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085774208:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829186.184009:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829186.184009:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829186.184021:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829186.184079:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829187.207928:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:3.0:1787829187.207932:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829187.208009:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787829187.208063:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829188.232031:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829188.232032:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787829188.232097:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829188.232235:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000400:00000100:3.0:1787829189.128131:0:2399:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.141:988 00000400:00000100:3.0:1787829189.128179:0:2399:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.141@tcp at host 192.168.203.141 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1787829189.128196:0:2399:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1787829189.128303:0:2399:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000800:00000100:3.0:1787829189.128326:0:2399:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:0.0:1787829189.128328:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829186/real 1787829189] req@ffff9e35da6f6d00 x1874672085773184/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829211 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:3.0:1787829189.128341:0:2399:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1787829189.128355:0:2399:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1787829189.128371:0:2399:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:0.0:1787829189.128374:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6d00 x1874672085773184/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829211 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:3.0:1787829189.128384:0:2399:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1787829189.128400:0:2399:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00080000:0.0:1787829189.128403:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829189.128413:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000800:00000100:3.0:1787829189.128414:0:2399:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:0.0:1787829189.128422:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085773184:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1787829189.128621:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829186/real 1787829189] req@ffff9e35da6f6580 x1874672085773824/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829211 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829189.128671:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6580 x1874672085773824/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829211 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829189.128686:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829189.128692:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829189.128699:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085773824:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829189.128708:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829186/real 1787829189] req@ffff9e35da6f4780 x1874672085773952/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829211 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829189.128723:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4780 x1874672085773952/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829211 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829189.128732:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829189.128736:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829189.128740:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085773952:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829189.128749:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829186/real 1787829189] req@ffff9e35da6f7480 x1874672085774080/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829211 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829189.128763:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7480 x1874672085774080/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829211 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829189.128773:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829189.128776:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829189.128780:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085774080:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829189.128787:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829186/real 1787829189] req@ffff9e35da6f4000 x1874672085774208/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829211 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829189.128878:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4000 x1874672085774208/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829211 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829189.128887:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829189.128891:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829189.128896:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085774208:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829189.128920:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829186/real 1787829189] req@ffff9e35da6f6940 x1874672085773312/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829211 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829189.128936:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6940 x1874672085773312/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829211 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829189.128946:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829189.128950:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829189.128953:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085773312:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829189.128961:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829186/real 1787829189] req@ffff9e35da6f4f00 x1874672085773440/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829211 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829189.128975:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4f00 x1874672085773440/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829211 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829189.128983:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829189.128987:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829189.128990:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085773440:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829189.128998:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829186/real 1787829189] req@ffff9e35da6f61c0 x1874672085773568/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829211 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829189.129012:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f61c0 x1874672085773568/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829211 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829189.129022:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829189.129025:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829189.129029:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085773568:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829189.129037:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829186/real 1787829189] req@ffff9e35da6f43c0 x1874672085773696/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829211 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829189.129050:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f43c0 x1874672085773696/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829211 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829189.129060:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829189.129064:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829189.129067:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085773696:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829189.255926:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:3.0:1787829189.255926:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829189.255947:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829189.256018:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829190.279927:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829190.279927:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:3.0:1787829190.279927:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787829190.280033:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829191.175925:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829191.175940:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829191.175945:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829191.175959:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2626 00000100:00080000:1.0:1787829191.175974:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829191.175984:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787829191.176034:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5a40 x1874672085774336/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829191.176069:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829191.176074:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829191.176076:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829191.176080:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2626 00000100:00080000:1.0:1787829191.176084:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829191.176088:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829191.176105:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829191.176126:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7840 x1874672085774464/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829191.176144:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787829191.176148:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085774336:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787829191.176151:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829191.176158:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829191.176166:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2626 00000100:00080000:1.0:1787829191.176176:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829191.176183:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829191.176189:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829191.176207:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f70c0 x1874672085774592/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829191.176218:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829191.176229:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829191.176231:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829191.176237:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2626 00000100:00080000:1.0:1787829191.176255:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829191.176262:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:0.0:1787829191.176272:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:1.0:1787829191.176273:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:1.0:1787829191.176300:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4b40 x1874672085774720/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829191.176317:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829191.176322:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829191.176324:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829191.176332:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2626 00000100:00080000:1.0:1787829191.176339:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787829191.176344:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829191/real 1787829191] req@ffff9e35da6f5a40 x1874672085774336/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829216 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829191.176351:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829191.176375:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:0.0:1787829191.176391:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5a40 x1874672085774336/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829216 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829191.176404:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5e00 x1874672085774848/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829191.176416:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829191.176424:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829191.176429:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:1.0:1787829191.176433:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829191.176439:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787829191.176448:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085774336:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787829191.176449:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2626 00000100:00080000:1.0:1787829191.176461:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829191.176471:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829191.176479:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829191.176500:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5a40 x1874672085774976/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829191.176504:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085774464:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829191.176545:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787829191.176546:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829191.176555:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829191.176560:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829191.176568:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2626 00000100:00100000:0.0:1787829191.176576:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829191/real 1787829191] req@ffff9e35da6f7840 x1874672085774464/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829216 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829191.176578:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829191.176591:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829191.176604:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829191.176612:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7840 x1874672085774464/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829216 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829191.176630:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1787829191.176631:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f43c0 x1874672085775104/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829191.176643:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829191.176654:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085774464:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787829191.176662:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787829191.176669:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085774592:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787829191.176672:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829191.176680:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829191.176692:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2626 00000100:00080000:1.0:1787829191.176698:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829191.176706:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:0.0:1787829191.176715:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:1.0:1787829191.176719:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:0.0:1787829191.176735:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829191/real 1787829191] req@ffff9e35da6f70c0 x1874672085774592/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829216 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829191.176746:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f61c0 x1874672085775232/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829191.176764:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f70c0 x1874672085774592/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829216 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829191.176768:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829191.176785:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829191.176791:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829191.176795:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829191.176836:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2626 00000100:00080000:1.0:1787829191.176843:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829191.176849:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829191.176858:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00080000:0.0:1787829191.176876:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829191.176881:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4f00 x1874672085775360/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829191.176888:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085774592:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829191.176932:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085774720:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829191.176971:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829191.176989:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829191/real 1787829191] req@ffff9e35da6f4b40 x1874672085774720/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829216 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829191.177007:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4b40 x1874672085774720/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829216 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829191.177028:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829191.177032:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829191.177037:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085774720:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829191.177046:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085774848:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829191.177071:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829191.177087:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829191/real 1787829191] req@ffff9e35da6f5e00 x1874672085774848/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829216 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829191.177102:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5e00 x1874672085774848/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829216 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829191.177114:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829191.177120:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829191.177124:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085774848:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829191.177157:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085774976:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829191.177194:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829191.177208:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829191/real 1787829191] req@ffff9e35da6f5a40 x1874672085774976/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829216 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829191.177224:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5a40 x1874672085774976/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829216 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829191.177236:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829191.177241:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829191.177247:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085774976:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829191.177256:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085775104:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829191.177283:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829191.177298:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829191/real 1787829191] req@ffff9e35da6f43c0 x1874672085775104/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829216 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829191.177314:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f43c0 x1874672085775104/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829216 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829191.177327:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829191.177331:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829191.177337:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085775104:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829191.177346:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085775232:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829191.177373:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829191.177389:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829191/real 1787829191] req@ffff9e35da6f61c0 x1874672085775232/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829216 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829191.177404:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f61c0 x1874672085775232/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829216 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829191.177416:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829191.177421:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829191.177425:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085775232:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829191.177434:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085775360:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829191.177461:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829191.177478:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829191/real 1787829191] req@ffff9e35da6f4f00 x1874672085775360/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829216 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829191.177495:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4f00 x1874672085775360/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829216 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829191.177507:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829191.177511:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829191.177516:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085775360:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829191.303993:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787829191.303993:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829191.303995:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:0.0:1787829191.304101:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829192.327960:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829192.327963:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829192.327996:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829192.328002:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829193.351931:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829193.351937:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829193.351937:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829193.351987:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829194.375910:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829194.375966:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787829194.375981:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:0.0:1787829194.376083:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829195.399939:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829195.399943:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829195.399983:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:0.0:1787829195.400007:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829196.295970:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829196.295982:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829196.295988:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829196.296011:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2631 00000100:00080000:1.0:1787829196.296025:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829196.296052:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787829196.296096:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6940 x1874672085775488/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829196.296130:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829196.296135:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829196.296137:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829196.296141:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2631 00000100:00080000:1.0:1787829196.296145:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829196.296149:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829196.296160:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829196.296172:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4000 x1874672085775616/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829196.296189:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829196.296193:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829196.296195:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829196.296198:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2631 00000100:00080000:1.0:1787829196.296201:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829196.296211:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829196.296216:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829196.296227:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7480 x1874672085775744/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829196.296238:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829196.296243:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829196.296245:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829196.296250:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2631 00000100:00080000:1.0:1787829196.296253:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829196.296257:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829196.296262:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:1.0:1787829196.296275:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4780 x1874672085775872/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829196.296288:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829196.296295:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829196.296304:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787829196.296308:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085775488:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787829196.296319:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2631 00000100:00080000:1.0:1787829196.296331:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829196.296339:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829196.296350:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:1.0:1787829196.296377:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6580 x1874672085776000/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829196.296397:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829196.296401:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829196.296403:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829196.296409:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2631 00000100:00080000:1.0:1787829196.296419:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829196.296425:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829196.296431:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1787829196.296439:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829196.296455:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6d00 x1874672085776128/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829196.296467:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829196.296470:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829196.296472:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829196.296476:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2631 00000100:00080000:1.0:1787829196.296479:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829196.296483:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829196.296489:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829196.296501:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4f00 x1874672085776256/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829196.296519:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829196.296522:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829196.296524:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829196.296529:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2631 00000100:00080000:1.0:1787829196.296533:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829196.296539:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829196.296545:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:0.0:1787829196.296563:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829196/real 1787829196] req@ffff9e35da6f6940 x1874672085775488/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829221 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829196.296564:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f61c0 x1874672085776384/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829196.296604:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829196.296608:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829196.296610:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829196.296624:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2631 00000100:00080000:1.0:1787829196.296628:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829196.296631:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829196.296635:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:1.0:1787829196.296649:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f43c0 x1874672085776512/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829196.296710:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6940 x1874672085775488/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829221 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829196.296733:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829196.296746:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829196.296753:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085775488:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1787829196.296770:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085775616:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829196.296907:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829196.296929:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829196/real 1787829196] req@ffff9e35da6f4000 x1874672085775616/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829221 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829196.296947:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4000 x1874672085775616/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829221 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829196.296967:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829196.296972:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829196.296976:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085775616:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829196.296986:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085775744:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829196.297011:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829196.297024:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829196/real 1787829196] req@ffff9e35da6f7480 x1874672085775744/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829221 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829196.297049:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7480 x1874672085775744/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829221 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829196.297059:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829196.297063:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829196.297067:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085775744:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829196.297110:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085775872:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829196.297131:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829196.297144:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829196/real 1787829196] req@ffff9e35da6f4780 x1874672085775872/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829221 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829196.297159:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4780 x1874672085775872/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829221 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829196.297170:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829196.297173:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829196.297177:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085775872:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829196.297185:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085776000:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829196.297204:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829196.297215:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829196/real 1787829196] req@ffff9e35da6f6580 x1874672085776000/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829221 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829196.297229:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6580 x1874672085776000/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829221 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829196.297240:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829196.297244:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829196.297247:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085776000:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829196.297255:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085776128:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829196.297273:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829196.297284:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829196/real 1787829196] req@ffff9e35da6f6d00 x1874672085776128/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829221 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829196.297307:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6d00 x1874672085776128/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829221 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829196.297326:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829196.297330:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829196.297334:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085776128:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829196.297341:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085776256:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829196.297361:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829196.297372:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829196/real 1787829196] req@ffff9e35da6f4f00 x1874672085776256/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829221 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829196.297386:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4f00 x1874672085776256/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829221 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829196.297396:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829196.297400:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829196.297403:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085776256:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829196.297410:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085776384:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829196.297430:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829196.297446:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829196/real 1787829196] req@ffff9e35da6f61c0 x1874672085776384/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829221 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829196.297470:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f61c0 x1874672085776384/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829221 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829196.297487:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829196.297493:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829196.297501:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085776384:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829196.297515:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085776512:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829196.297554:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829196.297600:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829196/real 1787829196] req@ffff9e35da6f43c0 x1874672085776512/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829221 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829196.297623:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f43c0 x1874672085776512/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829221 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829196.297640:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829196.297647:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829196.297653:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085776512:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829196.423971:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829196.423972:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829196.423980:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829196.424066:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829197.447999:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829197.448033:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829197.448034:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787829197.448142:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829198.471951:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829198.471982:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829198.471982:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829198.472110:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829199.496009:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829199.496014:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829199.496036:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829199.496163:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829200.519961:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829200.519972:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829200.520006:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:0.0:1787829200.520119:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829201.415922:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829201.415945:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787829201.415949:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829201.415957:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2636 00000100:00080000:0.0:1787829201.415967:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829201.415973:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787829201.416007:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7192c0 x1874672085776640/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829201.416036:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829201.416039:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829201.416040:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829201.416043:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2636 00000100:00080000:0.0:1787829201.416046:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829201.416048:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829201.416056:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829201.416069:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718780 x1874672085776768/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829201.416077:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829201.416080:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787829201.416097:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829201.416099:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2636 00000100:00080000:0.0:1787829201.416102:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787829201.416106:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085776640:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1787829201.416107:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829201.416115:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829201.416124:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a1c0 x1874672085776896/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829201.416132:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829201.416134:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829201.416136:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829201.416138:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2636 00000100:00080000:0.0:1787829201.416141:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829201.416144:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829201.416148:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:0.0:1787829201.416155:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7183c0 x1874672085777024/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829201.416162:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829201.416164:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829201.416166:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829201.416168:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2636 00000100:00080000:0.0:1787829201.416170:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829201.416172:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829201.416175:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:0.0:1787829201.416183:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a940 x1874672085777152/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829201.416185:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787829201.416194:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829201.416198:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829201.416200:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829201.416205:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2636 00000100:00080000:0.0:1787829201.416211:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829201.416216:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829201.416221:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829201.416235:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829201/real 1787829201] req@ffff9e35da7192c0 x1874672085776640/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829226 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829201.416237:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718f00 x1874672085777280/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829201.416264:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829201.416269:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:2.0:1787829201.416269:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7192c0 x1874672085776640/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829226 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829201.416271:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829201.416278:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2636 00000100:00080000:0.0:1787829201.416301:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829201.416308:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:2.0:1787829201.416310:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:0.0:1787829201.416312:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1787829201.416321:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829201.416326:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718000 x1874672085777408/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829201.416331:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085776640:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1787829201.416343:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787829201.416346:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085776768:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787829201.416347:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829201.416352:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829201.416356:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2636 00000100:00080000:0.0:1787829201.416360:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829201.416363:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829201.416367:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000800:00000100:2.0:1787829201.416379:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829201.416380:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a580 x1874672085777536/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829201.416390:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829201.416392:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829201.416394:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787829201.416398:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829201/real 1787829201] req@ffff9e35da718780 x1874672085776768/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829226 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829201.416399:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2636 00000100:00080000:0.0:1787829201.416404:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829201.416410:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829201.416417:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:0.0:1787829201.416429:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da719a40 x1874672085777664/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829201.416435:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718780 x1874672085776768/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829226 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829201.416472:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829201.416476:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829201.416478:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085776768:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829201.416503:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085776896:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787829201.416536:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829201.416545:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829201/real 1787829201] req@ffff9e35da71a1c0 x1874672085776896/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829226 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829201.416554:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a1c0 x1874672085776896/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829226 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829201.416559:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829201.416562:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829201.416564:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085776896:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829201.416567:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085777024:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829201.416579:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829201.416585:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829201/real 1787829201] req@ffff9e35da7183c0 x1874672085777024/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829226 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829201.416593:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672085777024/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829226 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829201.416599:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829201.416601:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829201.416603:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085777024:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829201.416606:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085777152:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829201.416616:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829201.416622:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829201/real 1787829201] req@ffff9e35da71a940 x1874672085777152/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829226 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829201.416646:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672085777152/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829226 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829201.416652:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829201.416654:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829201.416656:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085777152:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829201.416660:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085777280:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787829201.416670:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829201.416676:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829201/real 1787829201] req@ffff9e35da718f00 x1874672085777280/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829226 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829201.416683:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672085777280/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829226 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829201.416688:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829201.416706:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829201.416708:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085777280:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829201.416712:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085777408:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787829201.416724:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829201.416730:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829201/real 1787829201] req@ffff9e35da718000 x1874672085777408/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829226 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829201.416737:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718000 x1874672085777408/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829226 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829201.416742:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829201.416744:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829201.416746:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085777408:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829201.416749:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085777536:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829201.416759:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829201.416764:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829201/real 1787829201] req@ffff9e35da71a580 x1874672085777536/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829226 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829201.416771:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672085777536/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829226 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829201.416777:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829201.416779:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829201.416781:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085777536:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829201.416784:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085777664:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829201.416794:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829201.416800:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829201/real 1787829201] req@ffff9e35da719a40 x1874672085777664/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829226 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829201.416807:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672085777664/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829226 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829201.416813:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829201.416814:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829201.416816:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085777664:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829201.543904:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829201.543923:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829201.543939:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829201.543974:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829202.567875:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829202.567885:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829202.567892:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829202.567944:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829203.591960:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829203.591994:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829203.592011:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829203.592101:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829204.615984:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829204.615990:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829204.615993:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829204.616143:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829205.639902:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829205.639902:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829205.639905:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829205.639968:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829206.535893:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829206.535901:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829206.535905:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829206.535914:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2641 00000100:00080000:1.0:1787829206.535932:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829206.535941:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787829206.535974:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5a40 x1874672085777792/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829206.535995:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829206.535998:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829206.536002:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829206.536005:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2641 00000100:00080000:1.0:1787829206.536007:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829206.536010:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829206.536019:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829206.536028:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5e00 x1874672085777920/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829206.536037:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829206.536040:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:2.0:1787829206.536040:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085777792:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787829206.536044:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829206.536050:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2641 00000100:00080000:1.0:1787829206.536052:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829206.536055:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829206.536059:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829206.536068:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4b40 x1874672085778048/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829206.536074:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829206.536077:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829206.536078:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829206.536081:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2641 00000100:00080000:1.0:1787829206.536083:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829206.536085:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829206.536089:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:1.0:1787829206.536096:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f70c0 x1874672085778176/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829206.536102:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829206.536105:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829206.536106:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829206.536108:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2641 00000100:00080000:1.0:1787829206.536111:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1787829206.536114:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787829206.536118:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829206.536123:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:1.0:1787829206.536132:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7840 x1874672085778304/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829206.536139:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829206.536142:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829206.536143:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829206.536145:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2641 00000100:00080000:1.0:1787829206.536148:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829206.536150:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829206.536153:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829206.536160:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829206/real 1787829206] req@ffff9e35da6f5a40 x1874672085777792/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829231 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829206.536164:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f43c0 x1874672085778432/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829206.536187:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829206.536190:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829206.536191:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787829206.536196:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5a40 x1874672085777792/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829231 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829206.536198:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2641 00000100:00080000:1.0:1787829206.536203:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829206.536210:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829206.536215:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1787829206.536216:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829206.536223:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829206.536227:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f61c0 x1874672085778560/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829206.536228:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085777792:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787829206.536247:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829206.536250:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829206.536252:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829206.536256:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2641 00000100:00080000:1.0:1787829206.536259:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787829206.536261:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085777920:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787829206.536265:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829206.536270:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:1.0:1787829206.536281:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4f00 x1874672085778688/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829206.536289:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787829206.536290:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829206.536295:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829206.536296:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829206.536300:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2641 00000100:00080000:1.0:1787829206.536303:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787829206.536305:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829206/real 1787829206] req@ffff9e35da6f5e00 x1874672085777920/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829231 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829206.536308:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829206.536316:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:2.0:1787829206.536322:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5e00 x1874672085777920/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829231 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829206.536328:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6d00 x1874672085778816/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829206.536332:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829206.536340:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829206.536346:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085777920:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829206.536351:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085778048:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787829206.536372:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829206.536379:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829206/real 1787829206] req@ffff9e35da6f4b40 x1874672085778048/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829231 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829206.536387:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4b40 x1874672085778048/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829231 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829206.536393:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829206.536395:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829206.536397:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085778048:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829206.536401:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085778176:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829206.536412:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829206.536418:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829206/real 1787829206] req@ffff9e35da6f70c0 x1874672085778176/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829231 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829206.536425:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f70c0 x1874672085778176/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829231 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829206.536431:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829206.536433:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829206.536435:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085778176:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829206.536438:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085778304:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829206.536447:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829206.536453:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829206/real 1787829206] req@ffff9e35da6f7840 x1874672085778304/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829231 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829206.536460:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7840 x1874672085778304/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829231 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829206.536465:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829206.536467:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829206.536470:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085778304:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829206.536473:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085778432:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787829206.536483:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829206.536489:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829206/real 1787829206] req@ffff9e35da6f43c0 x1874672085778432/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829231 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829206.536495:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f43c0 x1874672085778432/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829231 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829206.536501:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829206.536502:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829206.536505:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085778432:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829206.536509:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085778560:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787829206.536519:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829206.536530:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829206/real 1787829206] req@ffff9e35da6f61c0 x1874672085778560/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829231 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829206.536538:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f61c0 x1874672085778560/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829231 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829206.536544:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829206.536546:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829206.536548:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085778560:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829206.536567:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085778688:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829206.536578:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829206.536585:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829206/real 1787829206] req@ffff9e35da6f4f00 x1874672085778688/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829231 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829206.536592:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4f00 x1874672085778688/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829231 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829206.536598:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829206.536600:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829206.536602:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085778688:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829206.536606:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085778816:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829206.536615:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829206.536621:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829206/real 1787829206] req@ffff9e35da6f6d00 x1874672085778816/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829231 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829206.536628:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6d00 x1874672085778816/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829231 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829206.536633:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829206.536635:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829206.536637:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085778816:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829206.663867:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829206.663867:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829206.663931:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829206.664006:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829207.687940:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829207.687948:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829207.687976:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:1.0:1787829207.687999:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829208.711894:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829208.711896:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829208.711907:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829208.711924:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829209.735903:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829209.735921:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829209.735949:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829209.735952:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829210.759949:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829210.759950:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829210.759972:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:1.0:1787829210.760015:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829211.655943:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829211.655962:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787829211.655967:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829211.655981:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2647 00000100:00080000:0.0:1787829211.655994:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829211.656004:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787829211.656049:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718b40 x1874672085778944/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829211.656078:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829211.656082:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829211.656084:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829211.656087:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2647 00000100:00080000:0.0:1787829211.656092:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829211.656097:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829211.656107:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829211.656117:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085778944:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1787829211.656121:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da719a40 x1874672085779072/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829211.656134:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829211.656137:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787829211.656139:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829211.656142:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2647 00000100:00080000:0.0:1787829211.656144:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829211.656148:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829211.656151:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829211.656160:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a580 x1874672085779200/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829211.656168:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829211.656171:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829211.656172:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829211.656175:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2647 00000100:00080000:0.0:1787829211.656192:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829211.656195:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829211.656198:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000800:00000100:1.0:1787829211.656206:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829211.656208:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718000 x1874672085779328/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829211.656221:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829211.656224:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829211.656226:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829211.656229:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2647 00000100:00080000:0.0:1787829211.656233:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829211.656235:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829211.656238:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:0.0:1787829211.656251:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718f00 x1874672085779456/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829211.656262:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829211.656265:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829211.656268:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787829211.656270:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829211/real 1787829211] req@ffff9e35da718b40 x1874672085778944/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829236 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829211.656276:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2647 00000100:00080000:0.0:1787829211.656282:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829211.656288:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829211.656293:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829211.656298:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718b40 x1874672085778944/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829236 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829211.656303:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a940 x1874672085779584/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829211.656324:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829211.656327:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829211.656330:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:0.0:1787829211.656335:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:1.0:1787829211.656338:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085778944:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1787829211.656341:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829211.656349:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2647 00000100:00080000:0.0:1787829211.656355:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829211.656359:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787829211.656365:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085779072:192.168.203.141@tcp:38:kworker.0 00000008:00100000:0.0:1787829211.656366:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829211.656384:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7183c0 x1874672085779712/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787829211.656392:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787829211.656395:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829211.656399:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829211.656401:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829211.656406:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2647 00000100:00100000:1.0:1787829211.656409:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829211/real 1787829211] req@ffff9e35da719a40 x1874672085779072/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829236 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829211.656411:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829211.656419:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829211.656426:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:1.0:1787829211.656435:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672085779072/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829236 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829211.656436:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a1c0 x1874672085779840/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829211.656468:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829211.656469:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829211.656476:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829211.656476:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:0.0:1787829211.656480:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787829211.656483:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085779072:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787829211.656487:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2647 00000100:00080000:0.0:1787829211.656492:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829211.656496:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787829211.656501:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085779200:192.168.203.141@tcp:38:kworker.0 00000008:00100000:0.0:1787829211.656502:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:0.0:1787829211.656512:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718780 x1874672085779968/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787829211.656532:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829211.656550:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829211/real 1787829211] req@ffff9e35da71a580 x1874672085779200/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829236 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829211.656565:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672085779200/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829236 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829211.656575:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829211.656578:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829211.656580:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085779200:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829211.656585:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085779328:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829211.656604:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829211.656612:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829211/real 1787829211] req@ffff9e35da718000 x1874672085779328/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829236 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829211.656621:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718000 x1874672085779328/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829236 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829211.656627:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829211.656629:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829211.656631:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085779328:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829211.656635:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085779456:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829211.656648:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829211.656656:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829211/real 1787829211] req@ffff9e35da718f00 x1874672085779456/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829236 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829211.656669:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672085779456/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829236 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829211.656679:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829211.656684:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829211.656700:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085779456:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829211.656708:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085779584:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787829211.656741:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829211.656771:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829211/real 1787829211] req@ffff9e35da71a940 x1874672085779584/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829236 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829211.656792:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672085779584/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829236 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829211.656845:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829211.656850:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829211.656854:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085779584:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829211.656886:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085779712:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787829211.656910:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829211.656921:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829211/real 1787829211] req@ffff9e35da7183c0 x1874672085779712/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829236 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829211.656932:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672085779712/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829236 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829211.656939:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829211.656942:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829211.656945:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085779712:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829211.656951:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085779840:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829211.656970:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829211.656981:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829211/real 1787829211] req@ffff9e35da71a1c0 x1874672085779840/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829236 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829211.656991:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a1c0 x1874672085779840/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829236 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829211.657002:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829211.657006:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829211.657008:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085779840:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829211.657014:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085779968:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829211.657035:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829211.657047:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829211/real 1787829211] req@ffff9e35da718780 x1874672085779968/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829236 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829211.657060:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718780 x1874672085779968/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829236 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829211.657070:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829211.657074:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829211.657077:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085779968:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829211.783930:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829211.783937:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829211.783988:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829211.784008:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829212.807961:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829212.807962:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829212.808001:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829212.808015:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829213.831908:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829213.831912:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829213.831926:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787829213.832096:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829214.855906:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829214.855919:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787829214.855977:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829214.855985:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829215.879906:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829215.879912:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829215.879948:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829215.879996:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829216.775894:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829216.775901:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829216.775905:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829216.775913:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2652 00000100:00080000:2.0:1787829216.775923:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829216.775929:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787829216.775961:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c000 x1874672085780096/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829216.775986:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829216.775989:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829216.775990:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829216.775993:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2652 00000100:00080000:2.0:1787829216.775995:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829216.775998:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829216.776006:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829216.776015:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cb40 x1874672085780224/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829216.776026:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829216.776034:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:1.0:1787829216.776035:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085780096:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1787829216.776037:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829216.776054:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2652 00000100:00080000:2.0:1787829216.776058:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829216.776064:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829216.776071:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829216.776085:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e1c0 x1874672085780352/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829216.776092:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829216.776096:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829216.776097:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829216.776101:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2652 00000100:00080000:2.0:1787829216.776105:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829216.776111:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829216.776115:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000800:00000100:1.0:1787829216.776127:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829216.776128:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f480 x1874672085780480/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829216.776140:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829216.776144:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829216.776146:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829216.776152:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2652 00000100:00080000:2.0:1787829216.776154:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829216.776158:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829216.776163:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:2.0:1787829216.776171:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f0c0 x1874672085780608/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829216.776178:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829216.776180:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829216.776182:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829216.776185:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2652 00000100:00080000:2.0:1787829216.776187:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1787829216.776192:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829216/real 1787829216] req@ffff9e35c498c000 x1874672085780096/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829241 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829216.776192:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829216.776199:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829216.776214:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e580 x1874672085780736/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829216.776256:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829216.776259:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829216.776260:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829216.776262:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2652 00000100:00080000:2.0:1787829216.776270:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829216.776275:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829216.776278:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829216.776288:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f840 x1874672085780864/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829216.776294:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c000 x1874672085780096/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829241 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829216.776303:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829216.776317:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829216.776327:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829216.776328:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829216.776334:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:2.0:1787829216.776335:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2652 00000100:00080000:2.0:1787829216.776342:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1787829216.776343:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085780096:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1787829216.776352:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829216.776358:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:2.0:1787829216.776370:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cf00 x1874672085780992/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829216.776379:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787829216.776383:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085780224:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1787829216.776387:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829216.776393:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829216.776398:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2652 00000100:00080000:2.0:1787829216.776401:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829216.776405:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829216.776408:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:2.0:1787829216.776417:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c3c0 x1874672085781120/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787829216.776426:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829216.776459:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829216/real 1787829216] req@ffff9e35c498cb40 x1874672085780224/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829241 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829216.776470:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498cb40 x1874672085780224/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829241 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829216.776478:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829216.776481:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829216.776483:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085780224:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829216.776488:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085780352:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787829216.776506:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829216.776513:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829216/real 1787829216] req@ffff9e35c498e1c0 x1874672085780352/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829241 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829216.776521:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e1c0 x1874672085780352/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829241 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829216.776532:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829216.776534:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829216.776537:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085780352:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829216.776541:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085780480:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829216.776552:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829216.776562:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829216/real 1787829216] req@ffff9e35c498f480 x1874672085780480/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829241 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829216.776575:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f480 x1874672085780480/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829241 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829216.776584:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829216.776587:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829216.776591:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085780480:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829216.776599:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085780608:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829216.776618:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829216.776626:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829216/real 1787829216] req@ffff9e35c498f0c0 x1874672085780608/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829241 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829216.776634:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f0c0 x1874672085780608/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829241 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829216.776641:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829216.776643:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829216.776647:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085780608:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829216.776651:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085780736:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787829216.776663:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829216.776669:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829216/real 1787829216] req@ffff9e35c498e580 x1874672085780736/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829241 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829216.776676:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e580 x1874672085780736/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829241 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829216.776682:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829216.776684:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829216.776685:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085780736:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829216.776689:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085780864:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787829216.776699:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829216.776705:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829216/real 1787829216] req@ffff9e35c498f840 x1874672085780864/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829241 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829216.776730:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f840 x1874672085780864/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829241 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829216.776736:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829216.776739:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829216.776741:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085780864:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829216.776761:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085780992:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829216.776776:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829216.776785:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829216/real 1787829216] req@ffff9e35c498cf00 x1874672085780992/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829241 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829216.776792:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498cf00 x1874672085780992/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829241 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829216.776819:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829216.776822:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829216.776824:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085780992:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829216.776828:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085781120:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829216.776841:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829216.776848:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829216/real 1787829216] req@ffff9e35c498c3c0 x1874672085781120/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829241 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829216.776856:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c3c0 x1874672085781120/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829241 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829216.776862:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829216.776864:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829216.776866:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085781120:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829216.903891:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829216.903927:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829216.903938:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787829216.903970:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829217.927891:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829217.927903:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829217.927913:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829217.927951:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829218.951930:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829218.951930:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:3.0:1787829218.951945:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829218.952023:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829219.975904:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829219.975955:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829219.975957:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787829219.975988:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829220.999912:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829220.999932:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829220.999933:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829220.999966:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829221.895945:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829221.895954:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787829221.895959:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829221.895968:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2657 00000100:00080000:3.0:1787829221.895978:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829221.895983:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787829221.896015:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499680 x1874672085781248/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829221.896042:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829221.896045:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829221.896046:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829221.896049:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2657 00000100:00080000:3.0:1787829221.896051:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829221.896060:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829221.896071:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829221.896080:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498f00 x1874672085781376/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829221.896087:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829221.896090:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787829221.896091:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829221.896094:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2657 00000100:00080000:3.0:1787829221.896097:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829221.896099:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829221.896102:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829221.896109:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b840 x1874672085781504/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829221.896116:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829221.896120:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:1.0:1787829221.896122:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085781248:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1787829221.896123:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829221.896134:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2657 00000100:00080000:3.0:1787829221.896138:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829221.896142:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829221.896148:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:3.0:1787829221.896163:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649bc00 x1874672085781632/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829221.896174:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829221.896178:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787829221.896180:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829221.896184:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2657 00000100:00080000:3.0:1787829221.896187:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1787829221.896190:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1787829221.896194:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829221.896199:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:3.0:1787829221.896211:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498b40 x1874672085781760/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829221.896217:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829221.896220:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829221.896221:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829221.896225:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2657 00000100:00080000:3.0:1787829221.896228:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1787829221.896232:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829221/real 1787829221] req@ffff9e35c6499680 x1874672085781248/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829246 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829221.896232:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829221.896239:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829221.896252:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a940 x1874672085781888/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829221.896256:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499680 x1874672085781248/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829246 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829221.896274:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829221.896277:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829221.896278:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829221.896280:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829221.896285:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:3.0:1787829221.896287:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2657 00000100:00080000:3.0:1787829221.896292:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1787829221.896293:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085781248:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1787829221.896297:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829221.896303:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829221.896306:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085781376:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787829221.896320:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498780 x1874672085782016/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787829221.896328:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1787829221.896338:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787829221.896341:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829221/real 1787829221] req@ffff9e35c6498f00 x1874672085781376/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829246 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829221.896345:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787829221.896350:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787829221.896356:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498f00 x1874672085781376/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829246 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829221.896359:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2657 00000100:00080000:3.0:1787829221.896366:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829221.896370:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829221.896373:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:1.0:1787829221.896377:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829221.896381:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085781376:192.168.203.141@tcp:38:kworker.0 00000008:00100000:3.0:1787829221.896381:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:3.0:1787829221.896398:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499a40 x1874672085782144/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829221.896406:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829221.896408:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:1.0:1787829221.896411:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085781504:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1787829221.896412:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829221.896420:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2657 00000100:00080000:3.0:1787829221.896424:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829221.896427:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829221.896431:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000800:00000100:1.0:1787829221.896435:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787829221.896444:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498f00 x1874672085782272/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829221.896448:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829221/real 1787829221] req@ffff9e35c649b840 x1874672085781504/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829246 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829221.896481:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b840 x1874672085781504/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829246 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829221.896488:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829221.896490:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829221.896493:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085781504:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829221.896497:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085781632:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829221.896508:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829221.896515:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829221/real 1787829221] req@ffff9e35c649bc00 x1874672085781632/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829246 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829221.896522:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649bc00 x1874672085781632/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829246 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829221.896527:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829221.896529:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829221.896531:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085781632:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829221.896534:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085781760:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829221.896544:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829221.896549:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829221/real 1787829221] req@ffff9e35c6498b40 x1874672085781760/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829246 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829221.896556:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498b40 x1874672085781760/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829246 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829221.896566:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829221.896568:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829221.896570:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085781760:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829221.896573:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085781888:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787829221.896582:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829221.896588:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829221/real 1787829221] req@ffff9e35c649a940 x1874672085781888/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829246 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829221.896595:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a940 x1874672085781888/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829246 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829221.896600:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829221.896602:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829221.896603:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085781888:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829221.896607:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085782016:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787829221.896616:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829221.896621:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829221/real 1787829221] req@ffff9e35c6498780 x1874672085782016/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829246 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829221.896629:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498780 x1874672085782016/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829246 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829221.896633:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829221.896635:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829221.896637:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085782016:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829221.896640:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085782144:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829221.896649:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829221.896654:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829221/real 1787829221] req@ffff9e35c6499a40 x1874672085782144/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829246 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829221.896661:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499a40 x1874672085782144/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829246 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829221.896666:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829221.896668:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829221.896670:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085782144:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829221.896697:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085782272:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829221.896709:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829221.896716:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829221/real 1787829221] req@ffff9e35c6498f00 x1874672085782272/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829246 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829221.896723:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498f00 x1874672085782272/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829246 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829221.896728:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829221.896730:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829221.896732:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085782272:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829222.023913:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829222.023931:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787829222.023932:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829222.024019:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829223.047921:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829223.047921:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787829223.047921:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787829223.048027:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829224.071888:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829224.071896:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829224.071909:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829224.071949:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829225.095898:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829225.095898:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829225.095903:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829225.095936:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829226.119864:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829226.119870:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829226.119906:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:1.0:1787829226.119940:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829227.015891:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829227.015903:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787829227.015908:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829227.015920:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2662 00000100:00080000:3.0:1787829227.015932:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829227.015947:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787829227.015984:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499680 x1874672085782400/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829227.016011:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829227.016015:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829227.016017:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829227.016021:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2662 00000100:00080000:3.0:1787829227.016026:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829227.016031:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829227.016042:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829227.016057:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a1c0 x1874672085782528/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829227.016065:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085782400:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1787829227.016075:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829227.016080:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787829227.016082:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829227.016087:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2662 00000100:00080000:3.0:1787829227.016090:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829227.016093:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829227.016099:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829227.016111:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c64983c0 x1874672085782656/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829227.016121:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829227.016125:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787829227.016127:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829227.016132:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2662 00000100:00080000:3.0:1787829227.016138:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829227.016142:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829227.016147:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000800:00000100:1.0:1787829227.016169:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787829227.016175:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649ad00 x1874672085782784/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829227.016189:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829227.016193:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787829227.016195:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829227.016201:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2662 00000100:00080000:3.0:1787829227.016206:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829227.016209:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829227.016215:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:1.0:1787829227.016230:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829227/real 1787829227] req@ffff9e35c6499680 x1874672085782400/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829252 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829227.016235:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498000 x1874672085782912/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829227.016267:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787829227.016273:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499680 x1874672085782400/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829252 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829227.016273:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829227.016278:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829227.016290:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2662 00000100:00080000:3.0:1787829227.016299:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829227.016307:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:1.0:1787829227.016309:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:3.0:1787829227.016314:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00080000:1.0:1787829227.016321:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829227.016327:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085782400:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1787829227.016342:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499e00 x1874672085783040/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829227.016353:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829227.016357:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787829227.016359:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829227.016364:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2662 00000100:00100000:1.0:1787829227.016369:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085782528:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1787829227.016371:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829227.016383:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829227.016390:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829227.016406:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c64992c0 x1874672085783168/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787829227.016412:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1787829227.016428:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787829227.016434:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829227/real 1787829227] req@ffff9e35c649a1c0 x1874672085782528/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829252 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829227.016435:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787829227.016441:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829227.016457:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2662 00000100:00080000:3.0:1787829227.016467:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829227.016472:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787829227.016478:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a1c0 x1874672085782528/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829252 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1787829227.016479:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00080000:1.0:1787829227.016500:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1787829227.016503:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b480 x1874672085783296/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829227.016508:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829227.016523:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085782528:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1787829227.016536:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787829227.016545:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c64983c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085782656:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1787829227.016546:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787829227.016550:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829227.016563:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2662 00000100:00080000:3.0:1787829227.016569:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829227.016574:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829227.016579:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000800:00000100:1.0:1787829227.016592:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787829227.016598:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b0c0 x1874672085783424/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829227.016613:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829227/real 1787829227] req@ffff9e35c64983c0 x1874672085782656/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829252 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829227.016629:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c64983c0 x1874672085782656/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829252 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829227.016638:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829227.016659:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829227.016664:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c64983c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085782656:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829227.016672:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085782784:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829227.016697:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829227.016710:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829227/real 1787829227] req@ffff9e35c649ad00 x1874672085782784/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829252 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829227.016722:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649ad00 x1874672085782784/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829252 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829227.016729:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829227.016732:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829227.016734:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085782784:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829227.016739:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085782912:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829227.016755:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829227.016763:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829227/real 1787829227] req@ffff9e35c6498000 x1874672085782912/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829252 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829227.016775:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498000 x1874672085782912/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829252 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829227.016782:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829227.016784:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829227.016786:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085782912:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829227.016790:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085783040:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787829227.016831:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829227.016841:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829227/real 1787829227] req@ffff9e35c6499e00 x1874672085783040/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829252 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829227.016850:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499e00 x1874672085783040/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829252 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829227.016855:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829227.016858:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829227.016860:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085783040:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829227.016878:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c64992c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085783168:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787829227.016890:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829227.016897:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829227/real 1787829227] req@ffff9e35c64992c0 x1874672085783168/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829252 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829227.016904:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c64992c0 x1874672085783168/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829252 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829227.016914:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829227.016917:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829227.016919:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c64992c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085783168:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829227.016923:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085783296:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829227.016933:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829227.016938:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829227/real 1787829227] req@ffff9e35c649b480 x1874672085783296/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829252 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829227.016945:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b480 x1874672085783296/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829252 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829227.016951:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829227.016953:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829227.016955:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085783296:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829227.016959:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085783424:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829227.016968:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829227.016974:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829227/real 1787829227] req@ffff9e35c649b0c0 x1874672085783424/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829252 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829227.016982:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b0c0 x1874672085783424/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829252 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829227.016987:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829227.016989:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829227.016991:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085783424:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829227.143905:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829227.143914:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829227.143936:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829227.143982:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829228.167919:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829228.167925:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829228.167926:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787829228.167989:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829229.191894:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829229.191894:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829229.191946:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829229.191959:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829230.215909:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829230.215922:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829230.215944:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829230.215948:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829231.239937:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829231.239939:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829231.239949:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829231.239960:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829232.135925:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829232.135935:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787829232.135939:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829232.135949:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2667 00000100:00080000:0.0:1787829232.135960:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829232.135968:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787829232.136006:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7192c0 x1874672085783552/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829232.136036:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829232.136042:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829232.136044:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829232.136049:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2667 00000100:00080000:0.0:1787829232.136053:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829232.136057:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829232.136068:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829232.136084:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085783552:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1787829232.136087:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718780 x1874672085783680/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829232.136102:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829232.136106:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787829232.136108:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829232.136113:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2667 00000100:00080000:0.0:1787829232.136117:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829232.136122:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829232.136127:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829232.136139:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a1c0 x1874672085783808/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787829232.136146:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787829232.136157:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829232.136161:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829232.136163:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829232.136169:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2667 00000100:00080000:0.0:1787829232.136174:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829232.136179:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829232.136185:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:1.0:1787829232.136185:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829232/real 1787829232] req@ffff9e35da7192c0 x1874672085783552/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829257 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829232.136207:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7183c0 x1874672085783936/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829232.136207:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7192c0 x1874672085783552/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829257 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829232.136223:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829232.136229:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829232.136230:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829232.136232:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829232.136236:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:0.0:1787829232.136243:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2667 00000100:00100000:1.0:1787829232.136243:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085783552:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1787829232.136252:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829232.136259:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829232.136264:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:1.0:1787829232.136272:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085783680:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829232.136279:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a940 x1874672085784064/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829232.136292:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829232.136296:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829232.136298:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1787829232.136305:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787829232.136307:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2667 00000100:00080000:0.0:1787829232.136313:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829232.136318:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787829232.136322:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829232/real 1787829232] req@ffff9e35da718780 x1874672085783680/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829257 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829232.136334:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718780 x1874672085783680/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829257 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1787829232.136337:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00080000:1.0:1787829232.136343:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829232.136348:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829232.136351:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085783680:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829232.136357:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718f00 x1874672085784192/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829232.136360:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085783808:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787829232.136376:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829232.136381:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000800:00000100:1.0:1787829232.136381:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787829232.136385:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787829232.136392:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829232/real 1787829232] req@ffff9e35da71a1c0 x1874672085783808/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829257 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829232.136394:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2667 00000100:00080000:0.0:1787829232.136401:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829232.136409:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829232.136415:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829232.136417:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a1c0 x1874672085783808/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829257 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829232.136430:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1787829232.136431:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718000 x1874672085784320/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829232.136439:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829232.136448:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085783808:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787829232.136451:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829232.136458:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829232.136460:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787829232.136462:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085783936:192.168.203.141@tcp:8:kworker.0 00000100:00080000:0.0:1787829232.136468:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2667 00000100:00080000:0.0:1787829232.136473:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829232.136477:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829232.136482:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000800:00000100:1.0:1787829232.136487:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829232.136497:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a580 x1874672085784448/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829232.136501:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829232/real 1787829232] req@ffff9e35da7183c0 x1874672085783936/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829257 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829232.136523:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787829232.136527:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672085783936/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829257 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829232.136530:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829232.136534:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829232.136541:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829232.136545:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2667 00000100:00080000:1.0:1787829232.136548:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:0.0:1787829232.136554:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1787829232.136554:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085783936:192.168.203.141@tcp:8:kworker.0 00000100:00080000:0.0:1787829232.136564:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829232.136570:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:1.0:1787829232.136576:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085784064:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829232.136585:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da719a40 x1874672085784576/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787829232.136594:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829232.136621:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829232/real 1787829232] req@ffff9e35da71a940 x1874672085784064/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829257 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829232.136631:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672085784064/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829257 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829232.136638:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829232.136641:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829232.136643:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085784064:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829232.136648:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085784192:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787829232.136673:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829232.136685:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829232/real 1787829232] req@ffff9e35da718f00 x1874672085784192/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829257 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829232.136698:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672085784192/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829257 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829232.136707:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829232.136710:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829232.136714:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085784192:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829232.136719:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085784320:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787829232.136734:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829232.136742:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829232/real 1787829232] req@ffff9e35da718000 x1874672085784320/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829257 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829232.136767:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718000 x1874672085784320/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829257 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829232.136777:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829232.136780:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829232.136783:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085784320:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829232.136790:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085784448:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829232.136833:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829232.136842:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829232/real 1787829232] req@ffff9e35da71a580 x1874672085784448/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829257 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829232.136849:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672085784448/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829257 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829232.136855:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829232.136857:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829232.136860:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085784448:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829232.136873:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085784576:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829232.136884:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829232.136891:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829232/real 1787829232] req@ffff9e35da719a40 x1874672085784576/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829257 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829232.136898:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672085784576/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829257 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829232.136907:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829232.136910:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829232.136919:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085784576:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829232.263959:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829232.263961:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829232.264012:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829232.264040:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829233.287956:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829233.287956:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829233.287957:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829233.287961:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829234.311915:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829234.311931:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829234.311999:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829234.312006:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829235.335921:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829235.335921:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829235.335922:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829235.335982:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829236.359892:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829236.359929:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829236.359951:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829236.359956:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829237.255926:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829237.255933:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829237.255937:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829237.255946:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2672 00000100:00080000:1.0:1787829237.255956:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829237.255963:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787829237.255998:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6580 x1874672085784704/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829237.256027:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829237.256030:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829237.256032:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829237.256034:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2672 00000100:00080000:1.0:1787829237.256037:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829237.256039:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829237.256048:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829237.256057:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4780 x1874672085784832/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829237.256065:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829237.256068:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829237.256069:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829237.256072:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2672 00000100:00080000:1.0:1787829237.256074:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829237.256077:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829237.256080:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829237.256085:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085784704:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1787829237.256088:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7480 x1874672085784960/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829237.256099:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829237.256101:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829237.256102:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829237.256105:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2672 00000100:00080000:1.0:1787829237.256107:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829237.256110:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829237.256113:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:1.0:1787829237.256128:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4000 x1874672085785088/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829237.256134:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829237.256139:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829237.256141:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829237.256146:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2672 00000800:00000100:2.0:1787829237.256150:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787829237.256153:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829237.256160:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829237.256166:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:2.0:1787829237.256187:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829237/real 1787829237] req@ffff9e35da6f6580 x1874672085784704/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829262 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829237.256190:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6940 x1874672085785216/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829237.256215:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787829237.256219:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6580 x1874672085784704/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829262 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829237.256221:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829237.256225:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829237.256232:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829237.256235:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2672 00000100:00080000:2.0:1787829237.256239:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:1.0:1787829237.256243:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787829237.256244:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085784704:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787829237.256252:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787829237.256255:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085784832:192.168.203.141@tcp:38:kworker.0 00000008:00100000:1.0:1787829237.256261:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829237.256277:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6d00 x1874672085785344/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829237.256285:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787829237.256294:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787829237.256299:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829237/real 1787829237] req@ffff9e35da6f4780 x1874672085784832/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829262 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829237.256300:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829237.256304:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829237.256314:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2672 00000100:00100000:2.0:1787829237.256319:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4780 x1874672085784832/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829262 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829237.256320:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829237.256329:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:2.0:1787829237.256333:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000008:00100000:1.0:1787829237.256336:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1787829237.256338:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829237.256344:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085784832:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829237.256354:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4f00 x1874672085785472/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829237.256365:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829237.256369:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829237.256371:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787829237.256376:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085784960:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787829237.256378:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2672 00000100:00080000:1.0:1787829237.256387:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829237.256391:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829237.256397:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:1.0:1787829237.256413:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f61c0 x1874672085785600/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829237.256413:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787829237.256428:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787829237.256430:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829237/real 1787829237] req@ffff9e35da6f7480 x1874672085784960/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829262 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829237.256434:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829237.256439:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829237.256449:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2672 00000100:00100000:2.0:1787829237.256450:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7480 x1874672085784960/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829262 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829237.256456:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829237.256466:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:2.0:1787829237.256468:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829237.256474:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000008:00100000:1.0:1787829237.256475:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:2.0:1787829237.256477:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085784960:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829237.256486:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085785088:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829237.256493:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f43c0 x1874672085785728/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829237.256504:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829237.256516:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829237/real 1787829237] req@ffff9e35da6f4000 x1874672085785088/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829262 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829237.256527:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4000 x1874672085785088/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829262 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829237.256533:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829237.256535:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829237.256537:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085785088:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829237.256540:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085785216:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829237.256550:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829237.256556:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829237/real 1787829237] req@ffff9e35da6f6940 x1874672085785216/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829262 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829237.256563:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6940 x1874672085785216/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829262 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829237.256569:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829237.256584:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829237.256587:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085785216:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829237.256591:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085785344:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787829237.256605:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829237.256612:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829237/real 1787829237] req@ffff9e35da6f6d00 x1874672085785344/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829262 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829237.256620:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6d00 x1874672085785344/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829262 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829237.256628:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829237.256632:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829237.256635:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085785344:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829237.256641:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085785472:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787829237.256659:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829237.256671:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829237/real 1787829237] req@ffff9e35da6f4f00 x1874672085785472/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829262 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829237.256682:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4f00 x1874672085785472/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829262 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829237.256687:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829237.256689:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829237.256691:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085785472:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829237.256710:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085785600:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829237.256723:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829237.256730:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829237/real 1787829237] req@ffff9e35da6f61c0 x1874672085785600/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829262 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829237.256738:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f61c0 x1874672085785600/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829262 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829237.256743:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829237.256745:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829237.256747:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085785600:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829237.256750:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085785728:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829237.256759:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829237.256765:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829237/real 1787829237] req@ffff9e35da6f43c0 x1874672085785728/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829262 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829237.256771:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f43c0 x1874672085785728/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829262 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829237.256777:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829237.256778:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829237.256780:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085785728:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829237.383869:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829237.383880:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829237.383895:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829237.383919:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829238.407901:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829238.407902:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829238.407913:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829238.407940:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829239.431893:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829239.431896:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829239.431899:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829239.431985:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829240.455946:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829240.455953:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829240.455967:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829240.456042:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829241.479897:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829241.479909:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829241.479909:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829241.479961:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829242.375926:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829242.375935:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829242.375939:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829242.375948:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2677 00000100:00080000:2.0:1787829242.375958:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829242.375964:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787829242.376007:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498d680 x1874672085785856/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829242.376045:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829242.376048:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829242.376049:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829242.376053:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2677 00000100:00080000:2.0:1787829242.376057:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829242.376061:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829242.376069:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829242.376081:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c780 x1874672085785984/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829242.376090:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829242.376093:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829242.376096:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829242.376104:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2677 00000100:00100000:1.0:1787829242.376105:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085785856:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1787829242.376107:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829242.376119:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829242.376127:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829242.376139:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e940 x1874672085786112/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829242.376158:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829242.376162:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829242.376165:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829242.376170:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2677 00000100:00080000:2.0:1787829242.376186:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829242.376190:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829242.376196:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:2.0:1787829242.376211:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498fc00 x1874672085786240/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829242.376220:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829242.376223:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829242.376224:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829242.376228:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2677 00000100:00080000:2.0:1787829242.376231:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829242.376234:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829242.376238:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000800:00000100:1.0:1787829242.376247:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829242.376255:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498de00 x1874672085786368/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829242.376263:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829242.376266:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829242.376267:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829242.376273:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2677 00000100:00080000:2.0:1787829242.376276:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829242.376279:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829242.376281:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829242.376289:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498ed00 x1874672085786496/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829242.376295:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829242.376297:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829242.376298:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829242.376301:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2677 00000100:00080000:2.0:1787829242.376304:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829242.376306:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829242.376311:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829242.376332:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c3c0 x1874672085786624/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829242.376339:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829242.376342:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829242.376344:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787829242.376353:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829242/real 1787829242] req@ffff9e35c498d680 x1874672085785856/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829267 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829242.376354:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2677 00000100:00080000:2.0:1787829242.376361:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829242.376369:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829242.376376:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:2.0:1787829242.376391:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cf00 x1874672085786752/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829242.376420:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787829242.376426:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498d680 x1874672085785856/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829267 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829242.376428:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829242.376435:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829242.376456:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2677 00000100:00080000:2.0:1787829242.376460:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829242.376461:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829242.376466:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829242.376484:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00080000:1.0:1787829242.376486:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829242.376494:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085785856:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1787829242.376501:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f840 x1874672085786880/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829242.376579:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085785984:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787829242.376634:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829242.376650:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829242/real 1787829242] req@ffff9e35c498c780 x1874672085785984/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829267 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829242.376664:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c780 x1874672085785984/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829267 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829242.376691:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829242.376694:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829242.376698:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085785984:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829242.376705:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085786112:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787829242.376722:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829242.376732:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829242/real 1787829242] req@ffff9e35c498e940 x1874672085786112/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829267 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829242.376765:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e940 x1874672085786112/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829267 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829242.376771:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829242.376774:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829242.376776:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085786112:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829242.376781:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085786240:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829242.376796:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829242.376806:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829242/real 1787829242] req@ffff9e35c498fc00 x1874672085786240/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829267 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829242.376821:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498fc00 x1874672085786240/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829267 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829242.376832:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829242.376836:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829242.376840:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085786240:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829242.376876:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085786368:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829242.376897:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829242.376906:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829242/real 1787829242] req@ffff9e35c498de00 x1874672085786368/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829267 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829242.376916:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498de00 x1874672085786368/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829267 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829242.376928:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829242.376931:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829242.376933:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085786368:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829242.376939:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085786496:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787829242.376952:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829242.376959:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829242/real 1787829242] req@ffff9e35c498ed00 x1874672085786496/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829267 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829242.376967:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498ed00 x1874672085786496/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829267 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829242.376974:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829242.376976:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829242.376978:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085786496:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829242.376982:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085786624:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787829242.376994:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829242.377001:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829242/real 1787829242] req@ffff9e35c498c3c0 x1874672085786624/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829267 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829242.377010:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c3c0 x1874672085786624/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829267 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829242.377016:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829242.377018:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829242.377021:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085786624:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829242.377026:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085786752:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829242.377038:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829242.377046:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829242/real 1787829242] req@ffff9e35c498cf00 x1874672085786752/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829267 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829242.377078:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498cf00 x1874672085786752/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829267 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829242.377090:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829242.377094:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829242.377099:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085786752:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829242.377109:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085786880:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829242.377137:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829242.377165:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829242/real 1787829242] req@ffff9e35c498f840 x1874672085786880/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829267 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829242.377181:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f840 x1874672085786880/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829267 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829242.377191:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829242.377195:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829242.377199:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085786880:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829242.504002:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829242.504003:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829242.504021:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829242.504089:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829243.527889:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829243.527889:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829243.527964:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829243.527994:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829244.551902:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829244.551904:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829244.551940:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829244.551999:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829245.575943:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829245.575945:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829245.575964:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829245.575972:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829246.599970:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829246.600009:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829246.600009:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829246.600094:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829247.495949:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829247.495971:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829247.495975:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829247.495984:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2682 00000100:00080000:2.0:1787829247.495995:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829247.496001:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787829247.496043:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e580 x1874672085787008/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829247.496065:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829247.496068:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829247.496087:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829247.496093:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2682 00000100:00080000:2.0:1787829247.496099:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829247.496107:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829247.496123:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829247.496137:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085787008:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1787829247.496164:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f0c0 x1874672085787136/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829247.496189:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829247.496195:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829247.496201:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829247.496208:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2682 00000100:00080000:2.0:1787829247.496219:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829247.496227:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829247.496235:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1787829247.496254:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829247.496260:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f480 x1874672085787264/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829247.496278:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829247.496283:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829247.496286:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829247.496292:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2682 00000100:00080000:2.0:1787829247.496312:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829247.496318:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787829247.496336:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829247/real 1787829247] req@ffff9e35c498e580 x1874672085787008/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829272 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1787829247.496345:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:1.0:1787829247.496369:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e580 x1874672085787008/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829272 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829247.496375:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e1c0 x1874672085787392/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829247.496406:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829247.496416:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:2.0:1787829247.496420:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787829247.496422:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085787008:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1787829247.496439:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829247.496442:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829247.496468:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2682 00000100:00080000:2.0:1787829247.496477:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1787829247.496479:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085787136:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1787829247.496496:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829247.496529:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000800:00000100:1.0:1787829247.496570:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829247.496576:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cb40 x1874672085787520/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829247.496592:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787829247.496594:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829247/real 1787829247] req@ffff9e35c498f0c0 x1874672085787136/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829272 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829247.496597:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829247.496606:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829247.496626:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2682 00000100:00080000:2.0:1787829247.496639:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829247.496655:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829247.496663:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829247.496667:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f0c0 x1874672085787136/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829272 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829247.496683:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c000 x1874672085787648/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829247.496684:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829247.496716:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829247.496725:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085787136:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1787829247.496725:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829247.496743:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:1.0:1787829247.496748:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085787264:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1787829247.496749:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829247.496768:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2682 00000100:00080000:2.0:1787829247.496774:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829247.496781:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829247.496788:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1787829247.496794:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829247.496806:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e580 x1874672085787776/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829247.496821:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829247/real 1787829247] req@ffff9e35c498f480 x1874672085787264/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829272 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829247.496836:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f480 x1874672085787264/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829272 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829247.496851:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829247.496855:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829247.496859:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829247.496864:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2682 00000100:00080000:2.0:1787829247.496869:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829247.496879:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:1.0:1787829247.496883:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000008:00100000:2.0:1787829247.496886:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00080000:1.0:1787829247.496892:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829247.496901:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085787264:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829247.496908:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f840 x1874672085787904/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829247.496920:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085787392:192.168.203.141@tcp:8:kworker.0 00000100:00080000:2.0:1787829247.496924:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829247.496930:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829247.496932:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829247.496940:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2682 00000100:00080000:2.0:1787829247.496946:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829247.496952:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829247.496959:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000800:00000100:1.0:1787829247.496974:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829247.496978:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cf00 x1874672085788032/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829247.497027:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829247/real 1787829247] req@ffff9e35c498e1c0 x1874672085787392/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829272 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829247.497051:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e1c0 x1874672085787392/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829272 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829247.497060:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829247.497064:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829247.497067:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085787392:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829247.497104:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085787520:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829247.497127:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829247.497137:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829247/real 1787829247] req@ffff9e35c498cb40 x1874672085787520/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829272 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829247.497153:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498cb40 x1874672085787520/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829272 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829247.497162:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829247.497164:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829247.497166:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085787520:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829247.497172:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085787648:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787829247.497186:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829247.497194:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829247/real 1787829247] req@ffff9e35c498c000 x1874672085787648/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829272 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829247.497204:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c000 x1874672085787648/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829272 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829247.497211:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829247.497214:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829247.497216:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085787648:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829247.497221:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085787776:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787829247.497234:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829247.497242:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829247/real 1787829247] req@ffff9e35c498e580 x1874672085787776/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829272 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829247.497251:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e580 x1874672085787776/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829272 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829247.497263:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829247.497266:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829247.497281:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085787776:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829247.497286:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085787904:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829247.497298:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829247.497305:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829247/real 1787829247] req@ffff9e35c498f840 x1874672085787904/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829272 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829247.497314:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f840 x1874672085787904/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829272 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829247.497321:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829247.497324:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829247.497326:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085787904:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829247.497331:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085788032:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829247.497353:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829247.497361:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829247/real 1787829247] req@ffff9e35c498cf00 x1874672085788032/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829272 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829247.497370:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498cf00 x1874672085788032/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829272 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829247.497377:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829247.497380:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829247.497383:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085788032:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829247.623939:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829247.623939:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829247.623954:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787829247.624073:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829248.647954:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829248.647986:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829248.648000:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787829248.648050:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829249.672000:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829249.672001:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829249.672106:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787829249.672113:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829250.695931:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829250.695940:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829250.695985:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829250.696004:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829251.719946:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829251.719968:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829251.719982:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829251.720008:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829252.615939:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829252.615950:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829252.615954:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829252.615965:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2687 00000100:00080000:2.0:1787829252.615977:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829252.615985:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787829252.616026:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c3c0 x1874672085788160/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829252.616056:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829252.616060:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829252.616062:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829252.616067:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2687 00000100:00080000:2.0:1787829252.616072:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829252.616076:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829252.616088:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829252.616104:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498ed00 x1874672085788288/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829252.616118:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085788160:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1787829252.616123:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829252.616132:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829252.616136:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829252.616143:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2687 00000100:00080000:2.0:1787829252.616148:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829252.616154:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829252.616161:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829252.616178:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498de00 x1874672085788416/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829252.616191:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829252.616195:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829252.616198:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829252.616203:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2687 00000100:00080000:2.0:1787829252.616208:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829252.616212:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829252.616218:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:2.0:1787829252.616232:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498fc00 x1874672085788544/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829252.616244:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829252.616248:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829252.616250:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829252.616259:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2687 00000100:00080000:2.0:1787829252.616264:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1787829252.616268:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085788288:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1787829252.616277:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829252.616286:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:1.0:1787829252.616315:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085788416:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829252.616323:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e940 x1874672085788672/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829252.616339:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085788544:192.168.203.141@tcp:8:kworker.0 00000100:00080000:2.0:1787829252.616340:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829252.616349:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829252.616353:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829252.616361:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2687 00000100:00080000:2.0:1787829252.616369:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1787829252.616372:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085788672:192.168.203.141@tcp:8:kworker.0 00000100:00080000:2.0:1787829252.616382:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829252.616390:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829252.616413:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c780 x1874672085788800/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829252.616435:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829252.616440:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829252.616444:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829252.616450:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2687 00000100:00080000:2.0:1787829252.616455:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1787829252.616458:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085788800:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1787829252.616475:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829252.616483:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829252.616501:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498d680 x1874672085788928/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829252.616516:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787829252.616527:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085788928:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1787829252.616529:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829252.616535:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829252.616544:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2687 00000100:00080000:2.0:1787829252.616548:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829252.616554:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829252.616562:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:2.0:1787829252.616580:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cf00 x1874672085789056/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829252.616596:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829252.616600:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829252.616602:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829252.616608:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2687 00000100:00080000:2.0:1787829252.616613:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829252.616620:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787829252.616629:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085789056:192.168.203.141@tcp:8:kworker.0 00000008:00100000:2.0:1787829252.616631:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:2.0:1787829252.616658:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f840 x1874672085789184/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829252.616692:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085789184:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829252.743905:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829252.743944:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829252.743965:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829252.743972:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829253.767900:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829253.767932:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829253.767933:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829253.767933:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829254.791950:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829254.791958:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829254.791967:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829254.792052:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1787829255.688030:0:2396:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.141:988 00000400:00000100:3.0:1787829255.688069:0:2396:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.141@tcp at host 192.168.203.141 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1787829255.688095:0:2396:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1787829255.688154:0:2396:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000800:00000100:3.0:1787829255.688168:0:2396:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1787829255.688177:0:2396:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1787829255.688185:0:2396:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1787829255.688190:0:2396:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1787829255.688195:0:2396:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1787829255.688199:0:2396:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1787829255.688204:0:2396:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:1.0:1787829255.688333:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829252/real 1787829255] req@ffff9e35c498f840 x1874672085789184/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829277 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829255.688379:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f840 x1874672085789184/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829277 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829255.688394:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829255.688400:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829255.688406:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085789184:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829255.688414:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829252/real 1787829255] req@ffff9e35c498cf00 x1874672085789056/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829277 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829255.688428:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498cf00 x1874672085789056/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829277 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829255.688438:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829255.688531:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829255.688536:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085789056:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829255.688545:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829252/real 1787829255] req@ffff9e35c498d680 x1874672085788928/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829277 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829255.688560:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498d680 x1874672085788928/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829277 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829255.688569:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829255.688573:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829255.688577:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085788928:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829255.688585:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829252/real 1787829255] req@ffff9e35c498c780 x1874672085788800/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829277 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829255.688599:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c780 x1874672085788800/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829277 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829255.688607:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829255.688611:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829255.688615:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085788800:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829255.688623:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829252/real 1787829255] req@ffff9e35c498e940 x1874672085788672/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829277 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829255.688636:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e940 x1874672085788672/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829277 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829255.688646:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829255.688650:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829255.688654:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085788672:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829255.688662:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829252/real 1787829255] req@ffff9e35c498ed00 x1874672085788288/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829277 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829255.688688:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498ed00 x1874672085788288/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829277 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829255.688697:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829255.688701:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829255.688704:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085788288:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829255.688718:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829252/real 1787829255] req@ffff9e35c498de00 x1874672085788416/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829277 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829255.688731:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498de00 x1874672085788416/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829277 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829255.688739:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829255.688743:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829255.688746:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085788416:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829255.688753:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829252/real 1787829255] req@ffff9e35c498fc00 x1874672085788544/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829277 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829255.688766:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498fc00 x1874672085788544/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829277 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829255.688775:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829255.688779:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829255.688782:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085788544:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829255.688789:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829252/real 1787829255] req@ffff9e35c498c3c0 x1874672085788160/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829277 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829255.688863:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c3c0 x1874672085788160/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829277 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829255.688872:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829255.688876:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829255.688879:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085788160:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1787829255.815958:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829255.815970:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829255.815991:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829255.815997:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829256.839894:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829256.839949:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829256.839980:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829256.839981:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829257.735934:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829257.735941:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787829257.735945:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829257.735953:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2693 00000100:00080000:0.0:1787829257.735962:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829257.735968:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787829257.735999:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718b40 x1874672085789312/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829257.736024:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829257.736027:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829257.736028:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829257.736031:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2693 00000100:00080000:0.0:1787829257.736033:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829257.736035:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829257.736044:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829257.736054:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da719a40 x1874672085789440/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829257.736065:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829257.736068:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787829257.736071:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829257.736076:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2693 00000100:00080000:0.0:1787829257.736080:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829257.736084:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829257.736096:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829257.736118:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a580 x1874672085789568/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829257.736118:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085789312:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1787829257.736145:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829257.736149:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829257.736153:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829257.736160:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2693 00000100:00080000:0.0:1787829257.736165:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829257.736170:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829257.736177:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:0.0:1787829257.736193:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718000 x1874672085789696/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829257.736203:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829257.736207:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829257.736209:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829257.736215:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2693 00000100:00080000:0.0:1787829257.736219:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1787829257.736231:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787829257.736232:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829257.736245:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:0.0:1787829257.736262:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718f00 x1874672085789824/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829257.736274:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829257.736278:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829257.736280:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829257.736286:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2693 00000100:00080000:0.0:1787829257.736293:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829257.736299:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829257.736306:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829257.736307:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829257/real 1787829257] req@ffff9e35da718b40 x1874672085789312/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829282 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829257.736329:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a940 x1874672085789952/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829257.736344:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718b40 x1874672085789312/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829282 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829257.736361:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829257.736374:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787829257.736377:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829257.736381:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829257.736394:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:0.0:1787829257.736399:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2693 00000100:00100000:1.0:1787829257.736408:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085789312:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1787829257.736409:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829257.736420:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829257.736447:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829257.736451:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085789440:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829257.736469:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7183c0 x1874672085790080/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829257.736484:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829257.736488:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829257.736491:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829257.736503:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2693 00000800:00000100:1.0:1787829257.736503:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787829257.736520:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829257.736526:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829257.736532:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:1.0:1787829257.736538:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829257/real 1787829257] req@ffff9e35da719a40 x1874672085789440/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829282 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829257.736554:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a1c0 x1874672085790208/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829257.736561:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672085789440/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829282 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829257.736603:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829257.736606:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829257.736613:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:0.0:1787829257.736617:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829257.736623:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787829257.736624:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085789440:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787829257.736646:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2693 00000100:00080000:0.0:1787829257.736654:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829257.736661:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829257.736668:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:1.0:1787829257.736679:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085789568:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829257.736690:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718780 x1874672085790336/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787829257.736719:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829257.736745:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829257/real 1787829257] req@ffff9e35da71a580 x1874672085789568/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829282 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829257.736760:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672085789568/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829282 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829257.736770:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829257.736774:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829257.736778:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085789568:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829257.736785:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085789696:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829257.736866:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829257.736880:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829257/real 1787829257] req@ffff9e35da718000 x1874672085789696/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829282 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829257.736895:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718000 x1874672085789696/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829282 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829257.736904:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829257.736908:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829257.736911:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085789696:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829257.736919:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085789824:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829257.736944:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829257.736959:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829257/real 1787829257] req@ffff9e35da718f00 x1874672085789824/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829282 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829257.736971:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672085789824/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829282 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829257.736981:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829257.736989:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829257.736993:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085789824:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829257.737003:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085789952:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787829257.737029:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829257.737045:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829257/real 1787829257] req@ffff9e35da71a940 x1874672085789952/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829282 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829257.737060:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672085789952/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829282 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829257.737071:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829257.737075:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829257.737078:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085789952:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829257.737086:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085790080:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787829257.737107:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829257.737119:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829257/real 1787829257] req@ffff9e35da7183c0 x1874672085790080/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829282 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829257.737132:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672085790080/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829282 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829257.737140:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829257.737143:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829257.737147:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085790080:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829257.737154:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085790208:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829257.737185:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829257.737198:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829257/real 1787829257] req@ffff9e35da71a1c0 x1874672085790208/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829282 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829257.737210:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a1c0 x1874672085790208/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829282 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829257.737219:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829257.737222:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829257.737225:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085790208:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829257.737255:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085790336:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829257.737277:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829257.737290:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829257/real 1787829257] req@ffff9e35da718780 x1874672085790336/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829282 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829257.737301:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718780 x1874672085790336/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829282 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829257.737310:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829257.737314:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829257.737317:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085790336:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829257.863923:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829257.863955:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829257.863975:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787829257.863984:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829258.887863:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829258.887881:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829258.887926:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787829258.887953:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829259.911943:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829259.912014:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:1.0:1787829259.912056:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829259.912075:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829260.935982:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:3.0:1787829260.936013:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787829260.936018:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829260.936054:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829261.959967:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829261.959971:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:3.0:1787829261.959977:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787829261.960139:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829262.855896:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829262.855908:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787829262.855912:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829262.855923:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2698 00000100:00080000:0.0:1787829262.855934:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829262.855942:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787829262.855986:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7192c0 x1874672085790464/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829262.856008:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829262.856011:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829262.856012:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829262.856015:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2698 00000100:00080000:0.0:1787829262.856018:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829262.856021:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829262.856032:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829262.856051:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718780 x1874672085790592/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829262.856051:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085790464:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1787829262.856070:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829262.856074:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787829262.856076:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829262.856080:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2698 00000100:00080000:0.0:1787829262.856083:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829262.856086:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829262.856091:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829262.856101:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a1c0 x1874672085790720/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829262.856108:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829262.856110:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829262.856112:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829262.856117:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2698 00000800:00000100:1.0:1787829262.856118:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787829262.856120:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829262.856125:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829262.856129:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:0.0:1787829262.856137:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7183c0 x1874672085790848/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829262.856143:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829262.856146:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829262.856147:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829262.856151:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2698 00000100:00080000:0.0:1787829262.856154:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829262.856159:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787829262.856160:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829262/real 1787829262] req@ffff9e35da7192c0 x1874672085790464/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829287 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1787829262.856164:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:0.0:1787829262.856177:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a940 x1874672085790976/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829262.856183:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7192c0 x1874672085790464/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829287 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829262.856187:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829262.856194:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829262.856198:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829262.856203:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2698 00000100:00080000:1.0:1787829262.856203:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829262.856210:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829262.856211:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829262.856218:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085790464:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1787829262.856223:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829262.856227:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829262.856236:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718f00 x1874672085791104/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829262.856243:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787829262.856246:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085790592:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787829262.856247:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787829262.856251:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829262.856257:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2698 00000100:00080000:0.0:1787829262.856260:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829262.856262:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829262.856265:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829262.856274:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718000 x1874672085791232/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787829262.856283:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787829262.856292:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829262.856295:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829262.856297:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829262.856302:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2698 00000100:00100000:1.0:1787829262.856303:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829262/real 1787829262] req@ffff9e35da718780 x1874672085790592/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829287 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829262.856307:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829262.856313:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829262.856319:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:1.0:1787829262.856323:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718780 x1874672085790592/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829287 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829262.856331:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a580 x1874672085791360/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829262.856332:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829262.856340:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:0.0:1787829262.856346:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787829262.856347:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085790592:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787829262.856350:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829262.856354:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787829262.856357:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085790720:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787829262.856358:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2698 00000100:00080000:0.0:1787829262.856364:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829262.856368:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829262.856371:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:0.0:1787829262.856380:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da719a40 x1874672085791488/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787829262.856383:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829262.856411:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829262/real 1787829262] req@ffff9e35da71a1c0 x1874672085790720/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829287 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829262.856423:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a1c0 x1874672085790720/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829287 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829262.856429:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829262.856431:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829262.856433:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085790720:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829262.856437:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085790848:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829262.856452:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829262.856459:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829262/real 1787829262] req@ffff9e35da7183c0 x1874672085790848/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829287 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829262.856466:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672085790848/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829287 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829262.856472:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829262.856473:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829262.856475:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085790848:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829262.856479:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085790976:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829262.856488:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829262.856494:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829262/real 1787829262] req@ffff9e35da71a940 x1874672085790976/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829287 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829262.856506:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672085790976/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829287 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829262.856511:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829262.856513:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829262.856515:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085790976:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829262.856527:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085791104:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787829262.856537:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829262.856544:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829262/real 1787829262] req@ffff9e35da718f00 x1874672085791104/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829287 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829262.856551:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672085791104/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829287 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829262.856556:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829262.856558:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829262.856560:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085791104:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829262.856563:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085791232:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787829262.856573:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829262.856578:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829262/real 1787829262] req@ffff9e35da718000 x1874672085791232/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829287 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829262.856585:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718000 x1874672085791232/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829287 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829262.856594:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829262.856596:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829262.856598:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085791232:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829262.856602:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085791360:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829262.856610:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829262.856616:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829262/real 1787829262] req@ffff9e35da71a580 x1874672085791360/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829287 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829262.856622:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672085791360/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829287 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829262.856628:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829262.856630:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829262.856631:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085791360:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829262.856635:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085791488:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829262.856643:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829262.856648:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829262/real 1787829262] req@ffff9e35da719a40 x1874672085791488/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829287 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829262.856655:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672085791488/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829287 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829262.856660:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829262.856662:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829262.856664:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085791488:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829262.983871:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829262.983883:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829262.983916:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787829262.983951:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829264.007899:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829264.007901:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829264.007960:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:0.0:1787829264.008117:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829265.031855:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829265.031919:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829265.031946:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829265.031961:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829266.055964:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829266.056057:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787829266.056069:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829266.056077:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829267.079870:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829267.079906:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829267.079914:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787829267.079988:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829267.976001:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829267.976008:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829267.976012:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829267.976020:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2703 00000100:00080000:2.0:1787829267.976030:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829267.976037:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787829267.976070:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e580 x1874672085791616/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829267.976098:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829267.976101:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829267.976102:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829267.976105:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2703 00000100:00080000:2.0:1787829267.976107:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829267.976110:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829267.976118:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829267.976126:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c000 x1874672085791744/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829267.976138:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829267.976141:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829267.976144:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829267.976150:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2703 00000100:00080000:2.0:1787829267.976152:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829267.976154:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829267.976158:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829267.976166:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cb40 x1874672085791872/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829267.976175:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829267.976178:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829267.976179:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829267.976181:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2703 00000100:00080000:2.0:1787829267.976185:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829267.976195:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829267.976200:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:2.0:1787829267.976207:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e1c0 x1874672085792000/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829267.976210:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085791616:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1787829267.976217:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829267.976229:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829267.976231:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829267.976235:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2703 00000100:00080000:2.0:1787829267.976239:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829267.976241:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829267.976244:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:2.0:1787829267.976253:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f480 x1874672085792128/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829267.976259:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829267.976262:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829267.976263:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829267.976266:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2703 00000100:00080000:2.0:1787829267.976269:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829267.976271:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829267.976274:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829267.976284:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f0c0 x1874672085792256/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829267.976289:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829267.976293:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829267.976294:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829267.976296:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2703 00000100:00080000:2.0:1787829267.976299:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829267.976302:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829267.976306:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829267.976315:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c3c0 x1874672085792384/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787829267.976328:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1787829267.976341:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829267.976347:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829267.976349:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829267.976355:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2703 00000100:00080000:2.0:1787829267.976373:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829267.976377:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829267.976382:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:2.0:1787829267.976391:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498fc00 x1874672085792512/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829267.976397:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829267.976400:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829267.976420:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829267.976427:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2703 00000100:00080000:2.0:1787829267.976433:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829267.976439:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787829267.976450:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829267/real 1787829267] req@ffff9e35c498e580 x1874672085791616/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829292 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1787829267.976454:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:2.0:1787829267.976502:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498de00 x1874672085792640/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829267.976518:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e580 x1874672085791616/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829292 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829267.976575:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829267.976586:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829267.976592:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085791616:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1787829267.976620:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085791744:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787829267.976662:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829267.976691:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829267/real 1787829267] req@ffff9e35c498c000 x1874672085791744/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829292 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829267.976706:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c000 x1874672085791744/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829292 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829267.976717:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829267.976722:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829267.976727:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085791744:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829267.976763:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085791872:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787829267.976786:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829267.976859:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829267/real 1787829267] req@ffff9e35c498cb40 x1874672085791872/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829292 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829267.976901:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498cb40 x1874672085791872/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829292 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829267.976919:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829267.976924:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829267.976928:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085791872:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829267.976937:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085792000:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829267.976969:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829267.976985:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829267/real 1787829267] req@ffff9e35c498e1c0 x1874672085792000/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829292 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829267.976999:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e1c0 x1874672085792000/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829292 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829267.977009:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829267.977013:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829267.977017:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085792000:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829267.977024:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085792128:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829267.977044:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829267.977058:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829267/real 1787829267] req@ffff9e35c498f480 x1874672085792128/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829292 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829267.977080:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f480 x1874672085792128/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829292 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829267.977097:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829267.977103:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829267.977109:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085792128:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829267.977122:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085792256:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787829267.977160:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829267.977175:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829267/real 1787829267] req@ffff9e35c498f0c0 x1874672085792256/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829292 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829267.977189:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f0c0 x1874672085792256/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829292 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829267.977200:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829267.977204:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829267.977208:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085792256:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829267.977215:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085792384:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787829267.977235:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829267.977246:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829267/real 1787829267] req@ffff9e35c498c3c0 x1874672085792384/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829292 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829267.977268:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c3c0 x1874672085792384/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829292 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829267.977277:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829267.977280:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829267.977284:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085792384:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829267.977290:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085792512:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829267.977307:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829267.977317:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829267/real 1787829267] req@ffff9e35c498fc00 x1874672085792512/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829292 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829267.977330:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498fc00 x1874672085792512/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829292 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829267.977339:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829267.977343:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829267.977346:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085792512:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829267.977353:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085792640:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829267.977407:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829267.977429:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829267/real 1787829267] req@ffff9e35c498de00 x1874672085792640/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829292 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829267.977443:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498de00 x1874672085792640/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829292 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829267.977465:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829267.977471:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829267.977477:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085792640:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829268.103909:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829268.103981:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829268.103999:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829268.104077:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829269.127951:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829269.127962:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829269.128018:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787829269.128103:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829270.151932:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829270.151963:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829270.152016:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:1.0:1787829270.152025:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829271.175939:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829271.175942:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829271.175946:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829271.176034:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829272.199908:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829272.199908:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:0.0:1787829272.199950:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829272.200065:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00080000:2.0:1787829273.095898:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829273.095906:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829273.095910:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829273.095919:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2708 00000100:00080000:2.0:1787829273.095928:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829273.095934:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787829273.095973:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498ed00 x1874672085792768/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829273.096001:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829273.096005:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829273.096006:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829273.096010:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2708 00000100:00080000:2.0:1787829273.096014:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829273.096024:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829273.096036:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829273.096037:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085792768:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1787829273.096056:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e940 x1874672085792896/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829273.096071:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829273.096074:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829273.096076:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829273.096081:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2708 00000100:00080000:2.0:1787829273.096086:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829273.096090:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829273.096096:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1787829273.096107:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829273.096111:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c780 x1874672085793024/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829273.096124:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829273.096128:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829273.096130:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829273.096135:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2708 00000100:00080000:2.0:1787829273.096140:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829273.096145:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787829273.096151:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829273/real 1787829273] req@ffff9e35c498ed00 x1874672085792768/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829298 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1787829273.096152:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:2.0:1787829273.096172:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498d680 x1874672085793152/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829273.096173:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498ed00 x1874672085792768/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829298 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829273.096196:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829273.096199:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829273.096200:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829273.096204:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829273.096207:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:2.0:1787829273.096211:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2708 00000100:00100000:1.0:1787829273.096215:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085792768:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1787829273.096217:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829273.096224:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829273.096227:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:2.0:1787829273.096235:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cf00 x1874672085793280/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829273.096243:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085792896:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1787829273.096244:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829273.096250:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829273.096252:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829273.096256:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2708 00000100:00080000:2.0:1787829273.096260:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829273.096263:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829273.096267:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1787829273.096271:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829273.096280:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f840 x1874672085793408/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829273.096286:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829273/real 1787829273] req@ffff9e35c498e940 x1874672085792896/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829298 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829273.096290:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829273.096299:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829273.096301:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829273.096308:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2708 00000100:00100000:1.0:1787829273.096312:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e940 x1874672085792896/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829298 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829273.096313:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829273.096318:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829273.096333:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829273.096342:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498ed00 x1874672085793536/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829273.096348:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829273.096350:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829273.096351:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829273.096356:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2708 00000100:00080000:2.0:1787829273.096361:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829273.096369:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829273.096369:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:1.0:1787829273.096378:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000008:00100000:2.0:1787829273.096379:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:1.0:1787829273.096382:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085792896:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829273.096390:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085793024:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829273.096396:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498de00 x1874672085793664/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787829273.096413:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1787829273.096421:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829273.096424:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:1.0:1787829273.096427:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829273/real 1787829273] req@ffff9e35c498c780 x1874672085793024/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829298 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829273.096427:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829273.096436:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2708 00000100:00080000:2.0:1787829273.096443:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1787829273.096447:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c780 x1874672085793024/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829298 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829273.096449:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829273.096458:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00080000:1.0:1787829273.096459:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829273.096467:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829273.096471:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498fc00 x1874672085793792/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829273.096473:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085793024:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829273.096489:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085793152:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829273.096515:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829273.096524:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829273/real 1787829273] req@ffff9e35c498d680 x1874672085793152/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829298 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829273.096532:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498d680 x1874672085793152/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829298 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829273.096543:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829273.096546:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829273.096548:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085793152:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829273.096563:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085793280:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829273.096575:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829273.096581:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829273/real 1787829273] req@ffff9e35c498cf00 x1874672085793280/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829298 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829273.096589:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498cf00 x1874672085793280/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829298 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829273.096595:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829273.096597:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829273.096599:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085793280:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829273.096602:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085793408:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787829273.096612:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829273.096618:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829273/real 1787829273] req@ffff9e35c498f840 x1874672085793408/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829298 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829273.096625:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f840 x1874672085793408/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829298 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829273.096630:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829273.096632:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829273.096634:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085793408:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829273.096638:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085793536:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787829273.096646:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829273.096652:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829273/real 1787829273] req@ffff9e35c498ed00 x1874672085793536/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829298 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829273.096658:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498ed00 x1874672085793536/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829298 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829273.096663:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829273.096665:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829273.096667:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085793536:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829273.096670:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085793664:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829273.096683:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829273.096693:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829273/real 1787829273] req@ffff9e35c498de00 x1874672085793664/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829298 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829273.096743:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498de00 x1874672085793664/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829298 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829273.096750:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829273.096752:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829273.096754:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085793664:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829273.096758:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085793792:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829273.096780:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829273.096788:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829273/real 1787829273] req@ffff9e35c498fc00 x1874672085793792/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829298 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829273.096795:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498fc00 x1874672085793792/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829298 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829273.096822:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829273.096824:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829273.096827:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085793792:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829273.223915:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829273.223938:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829273.224056:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829273.224082:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829274.247874:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829274.247888:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829274.247908:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829274.247926:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829275.271932:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829275.271995:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829275.272044:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787829275.272113:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829276.295920:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829276.295931:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829276.295966:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829276.295994:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829277.319979:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829277.320000:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829277.320000:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829277.320048:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829278.215982:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829278.215993:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829278.215998:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829278.216008:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2713 00000100:00080000:1.0:1787829278.216021:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829278.216028:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787829278.216070:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7840 x1874672085793920/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829278.216105:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829278.216109:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829278.216111:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829278.216115:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2713 00000100:00080000:1.0:1787829278.216136:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829278.216143:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829278.216156:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829278.216174:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085793920:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1787829278.216178:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f70c0 x1874672085794048/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829278.216218:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829278.216221:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829278.216224:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829278.216228:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2713 00000100:00080000:1.0:1787829278.216233:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829278.216237:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829278.216244:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829278.216259:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4b40 x1874672085794176/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829278.216270:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829278.216274:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829278.216276:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1787829278.216284:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787829278.216319:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2713 00000100:00080000:1.0:1787829278.216327:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829278.216332:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829278.216338:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:1.0:1787829278.216354:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5e00 x1874672085794304/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829278.216365:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829278/real 1787829278] req@ffff9e35da6f7840 x1874672085793920/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829303 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829278.216371:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829278.216381:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829278.216387:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829278.216402:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2713 00000100:00080000:1.0:1787829278.216409:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787829278.216420:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7840 x1874672085793920/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829303 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829278.216421:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829278.216434:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00080000:2.0:1787829278.216452:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829278.216461:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829278.216465:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5a40 x1874672085794432/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829278.216470:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085793920:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787829278.216495:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829278.216500:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829278.216502:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829278.216509:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2713 00000100:00100000:2.0:1787829278.216515:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085794048:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787829278.216517:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829278.216530:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829278.216538:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829278.216555:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7840 x1874672085794560/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829278.216563:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787829278.216580:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829278.216585:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829278.216590:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787829278.216590:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829278/real 1787829278] req@ffff9e35da6f70c0 x1874672085794048/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829303 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829278.216602:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2713 00000100:00080000:1.0:1787829278.216612:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829278.216619:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829278.216625:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829278.216626:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f70c0 x1874672085794048/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829303 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829278.216642:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f43c0 x1874672085794688/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829278.216689:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829278.216693:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829278.216695:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829278.216700:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829278.216705:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:1.0:1787829278.216715:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2713 00000100:00100000:2.0:1787829278.216716:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085794048:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787829278.216725:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787829278.216730:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085794176:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787829278.216735:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829278.216743:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:1.0:1787829278.216758:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f61c0 x1874672085794816/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829278.216766:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787829278.216778:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829278.216782:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:2.0:1787829278.216784:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829278/real 1787829278] req@ffff9e35da6f4b40 x1874672085794176/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829303 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829278.216786:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829278.216796:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2713 00000100:00080000:1.0:1787829278.216800:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829278.216802:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829278.216807:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:2.0:1787829278.216830:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4b40 x1874672085794176/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829303 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829278.216839:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829278.216843:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829278.216847:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085794176:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829278.216853:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085794304:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829278.216859:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4f00 x1874672085794944/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829278.216880:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829278.216895:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829278/real 1787829278] req@ffff9e35da6f5e00 x1874672085794304/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829303 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829278.216908:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5e00 x1874672085794304/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829303 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829278.216914:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829278.216917:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829278.216919:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085794304:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829278.216923:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085794432:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829278.216958:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829278.216968:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829278/real 1787829278] req@ffff9e35da6f5a40 x1874672085794432/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829303 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829278.216980:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5a40 x1874672085794432/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829303 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829278.216993:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829278.216998:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829278.217003:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085794432:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829278.217039:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085794560:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787829278.217069:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829278.217083:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829278/real 1787829278] req@ffff9e35da6f7840 x1874672085794560/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829303 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829278.217095:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7840 x1874672085794560/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829303 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829278.217105:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829278.217108:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829278.217111:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085794560:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829278.217117:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085794688:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787829278.217154:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829278.217167:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829278/real 1787829278] req@ffff9e35da6f43c0 x1874672085794688/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829303 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829278.217184:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f43c0 x1874672085794688/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829303 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829278.217215:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829278.217220:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829278.217225:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085794688:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829278.217233:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085794816:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829278.217255:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829278.217267:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829278/real 1787829278] req@ffff9e35da6f61c0 x1874672085794816/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829303 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829278.217278:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f61c0 x1874672085794816/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829303 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829278.217326:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829278.217329:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829278.217332:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085794816:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829278.217337:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085794944:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829278.217355:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829278.217364:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829278/real 1787829278] req@ffff9e35da6f4f00 x1874672085794944/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829303 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829278.217395:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4f00 x1874672085794944/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829303 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829278.217408:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829278.217413:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829278.217419:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085794944:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829278.343874:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829278.343883:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829278.343892:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829278.343909:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829279.367897:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829279.367900:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829279.367905:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829279.367964:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829280.391882:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829280.391882:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829280.391899:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829280.391975:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829281.415891:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829281.415897:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829281.415918:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829281.415962:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829282.439871:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829282.439874:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829282.439888:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787829282.439944:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829283.335939:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829283.335950:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829283.335955:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829283.335966:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2718 00000100:00080000:1.0:1787829283.335978:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829283.335987:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787829283.336029:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6d00 x1874672085795072/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829283.336066:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829283.336071:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829283.336073:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829283.336078:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2718 00000100:00080000:1.0:1787829283.336083:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829283.336088:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829283.336101:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829283.336115:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6940 x1874672085795200/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829283.336120:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085795072:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787829283.336134:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829283.336137:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829283.336139:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829283.336143:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2718 00000100:00080000:1.0:1787829283.336147:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829283.336151:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829283.336156:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829283.336166:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4000 x1874672085795328/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829283.336173:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829283.336182:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829283.336184:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829283.336187:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2718 00000100:00080000:1.0:1787829283.336191:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1787829283.336197:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787829283.336198:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829283.336206:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:1.0:1787829283.336216:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7480 x1874672085795456/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829283.336224:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829283.336227:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829283.336228:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829283.336232:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2718 00000100:00080000:1.0:1787829283.336234:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829283.336237:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829283.336240:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:1.0:1787829283.336284:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4780 x1874672085795584/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829283.336287:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829283/real 1787829283] req@ffff9e35da6f6d00 x1874672085795072/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829308 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829283.336307:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829283.336313:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829283.336316:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787829283.336322:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6d00 x1874672085795072/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829308 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829283.336323:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2718 00000100:00080000:1.0:1787829283.336329:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829283.336334:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829283.336338:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1787829283.336367:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1787829283.336371:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6580 x1874672085795712/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829283.336376:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:1.0:1787829283.336387:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787829283.336387:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085795072:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787829283.336395:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829283.336399:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829283.336404:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2718 00000100:00080000:1.0:1787829283.336407:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829283.336410:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829283.336413:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829283.336422:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4f00 x1874672085795840/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829283.336430:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787829283.336432:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085795200:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787829283.336435:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829283.336438:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829283.336445:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2718 00000100:00080000:1.0:1787829283.336447:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829283.336451:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829283.336459:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:1.0:1787829283.336474:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f61c0 x1874672085795968/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829283.336476:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787829283.336505:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787829283.336507:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829283/real 1787829283] req@ffff9e35da6f6940 x1874672085795200/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829308 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829283.336514:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829283.336519:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787829283.336523:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6940 x1874672085795200/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829308 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829283.336527:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2718 00000100:00080000:1.0:1787829283.336535:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829283.336538:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829283.336542:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:2.0:1787829283.336545:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000008:00100000:1.0:1787829283.336549:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:2.0:1787829283.336550:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085795200:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829283.336560:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085795328:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829283.336566:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f43c0 x1874672085796096/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829283.336579:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829283.336588:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829283/real 1787829283] req@ffff9e35da6f4000 x1874672085795328/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829308 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829283.336596:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4000 x1874672085795328/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829308 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829283.336603:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829283.336605:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829283.336609:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085795328:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829283.336612:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085795456:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829283.336628:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829283.336637:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829283/real 1787829283] req@ffff9e35da6f7480 x1874672085795456/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829308 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829283.336644:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7480 x1874672085795456/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829308 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829283.336650:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829283.336652:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829283.336656:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085795456:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829283.336661:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085795584:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829283.336675:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829283.336684:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829283/real 1787829283] req@ffff9e35da6f4780 x1874672085795584/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829308 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829283.336691:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4780 x1874672085795584/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829308 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829283.336697:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829283.336700:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829283.336702:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085795584:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829283.336706:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085795712:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787829283.336718:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829283.336728:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829283/real 1787829283] req@ffff9e35da6f6580 x1874672085795712/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829308 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829283.336741:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6580 x1874672085795712/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829308 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829283.336748:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829283.336751:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829283.336754:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085795712:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829283.336773:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085795840:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787829283.336789:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829283.336797:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829283/real 1787829283] req@ffff9e35da6f4f00 x1874672085795840/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829308 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829283.336805:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4f00 x1874672085795840/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829308 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829283.336811:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829283.336814:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829283.336816:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085795840:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829283.336820:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085795968:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829283.336872:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829283.336881:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829283/real 1787829283] req@ffff9e35da6f61c0 x1874672085795968/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829308 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829283.336890:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f61c0 x1874672085795968/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829308 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829283.336900:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829283.336902:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829283.336905:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085795968:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829283.336910:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085796096:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829283.336921:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829283.336928:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829283/real 1787829283] req@ffff9e35da6f43c0 x1874672085796096/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829308 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829283.336935:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f43c0 x1874672085796096/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829308 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829283.336941:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829283.336943:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829283.336947:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085796096:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829283.463946:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829283.463965:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829283.463967:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829283.463991:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829284.487887:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829284.487887:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829284.487887:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829284.487945:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829285.511973:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829285.512022:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829285.512031:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787829285.512073:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829286.535984:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829286.535986:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829286.535988:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829286.536054:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829287.559937:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829287.559940:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829287.559975:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829287.559985:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829288.455922:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829288.455930:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787829288.455934:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829288.455942:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2723 00000100:00080000:0.0:1787829288.455952:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829288.455957:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787829288.455989:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718b40 x1874672085796224/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829288.456016:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829288.456021:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829288.456023:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829288.456027:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2723 00000100:00080000:0.0:1787829288.456031:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829288.456036:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829288.456048:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829288.456063:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da719a40 x1874672085796352/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829288.456076:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787829288.456078:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085796224:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1787829288.456080:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787829288.456087:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829288.456094:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2723 00000100:00080000:0.0:1787829288.456097:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829288.456100:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829288.456105:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829288.456115:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a580 x1874672085796480/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829288.456122:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829288.456125:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829288.456126:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829288.456129:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2723 00000100:00080000:0.0:1787829288.456131:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829288.456133:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829288.456137:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:0.0:1787829288.456145:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718000 x1874672085796608/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829288.456152:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829288.456156:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829288.456157:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829288.456163:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2723 00000800:00000100:2.0:1787829288.456165:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787829288.456168:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829288.456174:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829288.456178:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:0.0:1787829288.456188:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718f00 x1874672085796736/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829288.456196:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829288.456199:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829288.456200:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829288.456203:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2723 00000100:00080000:0.0:1787829288.456205:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829288.456209:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829288.456212:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829288.456236:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a940 x1874672085796864/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829288.456243:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829288.456247:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:2.0:1787829288.456249:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829288/real 1787829288] req@ffff9e35da718b40 x1874672085796224/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829313 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829288.456250:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829288.456256:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2723 00000100:00080000:0.0:1787829288.456264:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829288.456271:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829288.456283:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829288.456299:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7183c0 x1874672085796992/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829288.456313:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829288.456316:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829288.456318:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787829288.456322:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718b40 x1874672085796224/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829313 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829288.456326:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2723 00000100:00080000:0.0:1787829288.456335:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829288.456349:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:2.0:1787829288.456353:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:0.0:1787829288.456358:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00080000:2.0:1787829288.456360:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829288.456370:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085796224:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1787829288.456383:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a1c0 x1874672085797120/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829288.456396:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829288.456399:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829288.456400:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829288.456405:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2723 00000100:00080000:0.0:1787829288.456408:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787829288.456413:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085796352:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787829288.456415:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829288.456423:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:0.0:1787829288.456435:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718780 x1874672085797248/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829288.456459:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829288.456477:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829288/real 1787829288] req@ffff9e35da719a40 x1874672085796352/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829313 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829288.456488:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672085796352/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829313 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829288.456495:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829288.456498:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829288.456500:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085796352:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829288.456505:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085796480:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787829288.456526:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829288.456539:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829288/real 1787829288] req@ffff9e35da71a580 x1874672085796480/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829313 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829288.456553:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672085796480/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829313 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829288.456569:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829288.456573:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829288.456577:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085796480:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829288.456585:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085796608:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829288.456607:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829288.456620:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829288/real 1787829288] req@ffff9e35da718000 x1874672085796608/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829313 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829288.456634:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718000 x1874672085796608/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829313 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829288.456643:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829288.456647:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829288.456651:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085796608:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829288.456658:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085796736:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829288.456677:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829288.456686:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829288/real 1787829288] req@ffff9e35da718f00 x1874672085796736/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829313 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829288.456694:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672085796736/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829313 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829288.456700:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829288.456702:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829288.456705:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085796736:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829288.456709:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085796864:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787829288.456722:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829288.456728:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829288/real 1787829288] req@ffff9e35da71a940 x1874672085796864/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829313 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829288.456735:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672085796864/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829313 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829288.456741:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829288.456743:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829288.456744:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085796864:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829288.456748:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085796992:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787829288.456759:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829288.456764:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829288/real 1787829288] req@ffff9e35da7183c0 x1874672085796992/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829313 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829288.456772:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672085796992/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829313 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829288.456777:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829288.456779:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829288.456781:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085796992:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829288.456784:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085797120:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829288.456794:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829288.456834:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829288/real 1787829288] req@ffff9e35da71a1c0 x1874672085797120/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829313 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829288.456843:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a1c0 x1874672085797120/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829313 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829288.456849:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829288.456851:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829288.456853:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085797120:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829288.456873:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085797248:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829288.456886:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829288.456893:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829288/real 1787829288] req@ffff9e35da718780 x1874672085797248/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829313 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829288.456900:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718780 x1874672085797248/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829313 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829288.456906:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829288.456908:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829288.456910:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085797248:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829288.583899:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829288.583903:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829288.583947:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:0.0:1787829288.584008:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829289.607923:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829289.607924:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829289.607988:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787829289.608004:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829290.631891:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829290.631891:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829290.631938:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829290.631950:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829291.655893:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829291.655927:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829291.655977:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:1.0:1787829291.655990:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829292.679873:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829292.679903:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829292.679955:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787829292.679957:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829293.576012:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829293.576022:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829293.576026:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829293.576037:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2728 00000100:00080000:2.0:1787829293.576048:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829293.576055:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787829293.576090:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c3c0 x1874672085797376/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829293.576125:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829293.576129:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829293.576131:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829293.576135:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2728 00000100:00080000:2.0:1787829293.576139:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829293.576142:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829293.576153:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829293.576166:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f0c0 x1874672085797504/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829293.576174:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085797376:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1787829293.576194:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829293.576200:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829293.576202:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829293.576208:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2728 00000100:00080000:2.0:1787829293.576212:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829293.576216:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829293.576231:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829293.576245:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f480 x1874672085797632/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829293.576262:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829293.576265:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829293.576267:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1787829293.576273:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1787829293.576274:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2728 00000100:00080000:2.0:1787829293.576280:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829293.576285:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829293.576290:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:2.0:1787829293.576304:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e1c0 x1874672085797760/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829293.576315:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787829293.576317:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829293/real 1787829293] req@ffff9e35c498c3c0 x1874672085797376/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829318 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829293.576321:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829293.576326:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829293.576335:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2728 00000100:00080000:2.0:1787829293.576340:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1787829293.576341:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c3c0 x1874672085797376/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829318 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829293.576348:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:1.0:1787829293.576354:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:2.0:1787829293.576356:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00080000:1.0:1787829293.576360:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829293.576364:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085797376:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1787829293.576373:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cb40 x1874672085797888/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829293.576385:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787829293.576389:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085797504:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1787829293.576391:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829293.576396:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829293.576403:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2728 00000100:00080000:2.0:1787829293.576409:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1787829293.576424:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1787829293.576427:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829293.576433:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829293.576438:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829293/real 1787829293] req@ffff9e35c498f0c0 x1874672085797504/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829318 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829293.576447:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c000 x1874672085798016/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829293.576450:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f0c0 x1874672085797504/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829318 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829293.576477:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829293.576482:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:2.0:1787829293.576482:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787829293.576487:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085797504:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1787829293.576489:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829293.576494:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787829293.576496:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085797632:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1787829293.576502:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2728 00000100:00080000:2.0:1787829293.576507:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829293.576511:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:1.0:1787829293.576515:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:2.0:1787829293.576519:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829293.576525:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829293/real 1787829293] req@ffff9e35c498f480 x1874672085797632/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829318 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829293.576536:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e580 x1874672085798144/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829293.576544:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f480 x1874672085797632/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829318 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829293.576554:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829293.576557:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829293.576559:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829293.576562:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:2.0:1787829293.576564:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787829293.576568:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085797632:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1787829293.576573:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2728 00000100:00100000:1.0:1787829293.576576:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085797760:192.168.203.141@tcp:8:kworker.0 00000100:00080000:2.0:1787829293.576579:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1787829293.576593:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1787829293.576595:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829293.576602:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:1.0:1787829293.576603:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829293/real 1787829293] req@ffff9e35c498e1c0 x1874672085797760/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829318 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829293.576615:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e1c0 x1874672085797760/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829318 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829293.576620:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c3c0 x1874672085798272/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829293.576632:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829293.576641:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829293.576647:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085797760:192.168.203.141@tcp:8:kworker.0 00000100:00080000:2.0:1787829293.576649:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787829293.576654:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085797888:192.168.203.141@tcp:8:kworker.0 00000100:00080000:2.0:1787829293.576655:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829293.576661:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829293.576667:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2728 00000100:00080000:2.0:1787829293.576671:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1787829293.576675:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1787829293.576680:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829293.576686:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:1.0:1787829293.576687:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829293/real 1787829293] req@ffff9e35c498cb40 x1874672085797888/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829318 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829293.576698:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498cb40 x1874672085797888/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829318 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829293.576703:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498fc00 x1874672085798400/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829293.576734:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829293.576739:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829293.576743:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085797888:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829293.576765:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085798016:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787829293.576789:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829293.576846:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829293/real 1787829293] req@ffff9e35c498c000 x1874672085798016/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829318 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829293.576856:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c000 x1874672085798016/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829318 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829293.576862:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829293.576864:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829293.576866:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085798016:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829293.576871:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085798144:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787829293.576885:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829293.576892:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829293/real 1787829293] req@ffff9e35c498e580 x1874672085798144/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829318 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829293.576899:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e580 x1874672085798144/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829318 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829293.576904:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829293.576906:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829293.576909:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085798144:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829293.576913:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085798272:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829293.576933:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829293.576944:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829293/real 1787829293] req@ffff9e35c498c3c0 x1874672085798272/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829318 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829293.576976:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c3c0 x1874672085798272/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829318 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829293.576987:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829293.576990:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829293.576993:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085798272:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829293.576998:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085798400:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829293.577012:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829293.577020:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829293/real 1787829293] req@ffff9e35c498fc00 x1874672085798400/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829318 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829293.577027:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498fc00 x1874672085798400/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829318 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829293.577033:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829293.577035:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829293.577037:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085798400:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829293.703898:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829293.703967:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829293.704078:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829293.704112:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829294.727944:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829294.727967:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829294.727982:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787829294.727994:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829295.752004:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829295.752021:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829295.752021:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829295.752090:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829296.775918:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829296.775929:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829296.775956:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829296.776009:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829297.799930:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829297.799990:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829297.799990:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829297.800054:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829298.695845:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829298.695852:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829298.695856:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829298.695865:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2734 00000100:00080000:2.0:1787829298.695875:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829298.695880:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787829298.695912:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498de00 x1874672085798528/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829298.695936:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829298.695939:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829298.695940:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829298.695944:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2734 00000100:00080000:2.0:1787829298.695946:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829298.695948:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829298.695957:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829298.695965:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498ed00 x1874672085798656/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829298.695971:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829298.695973:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829298.695974:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829298.695977:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2734 00000100:00080000:2.0:1787829298.695978:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829298.695980:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829298.695984:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829298.695991:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f840 x1874672085798784/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829298.695997:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829298.696000:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829298.696001:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829298.696004:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2734 00000100:00080000:2.0:1787829298.696006:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829298.696010:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829298.696013:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:2.0:1787829298.696022:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cf00 x1874672085798912/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829298.696028:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085798528:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1787829298.696033:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829298.696042:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829298.696043:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829298.696048:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2734 00000100:00080000:2.0:1787829298.696050:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829298.696051:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829298.696056:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:2.0:1787829298.696065:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498d680 x1874672085799040/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829298.696071:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829298.696074:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829298.696075:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829298.696079:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2734 00000100:00080000:2.0:1787829298.696083:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829298.696086:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829298.696089:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829298.696097:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c780 x1874672085799168/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829298.696103:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829298.696104:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829298.696106:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829298.696109:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2734 00000100:00080000:2.0:1787829298.696114:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1787829298.696121:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1787829298.696126:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829298.696131:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829298.696160:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e940 x1874672085799296/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829298.696168:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829298.696170:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829298.696172:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829298.696177:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2734 00000100:00080000:2.0:1787829298.696181:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829298.696185:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829298.696191:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:1.0:1787829298.696196:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829298/real 1787829298] req@ffff9e35c498de00 x1874672085798528/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829323 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829298.696207:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498fc00 x1874672085799424/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829298.696215:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498de00 x1874672085798528/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829323 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829298.696215:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829298.696234:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829298.696242:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829298.696252:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829298.696253:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2734 00000100:00080000:1.0:1787829298.696270:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829298.696282:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085798528:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1787829298.696286:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1787829298.696292:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085798656:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1787829298.696300:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829298.696312:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000800:00000100:1.0:1787829298.696334:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829298.696346:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829298/real 1787829298] req@ffff9e35c498ed00 x1874672085798656/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829323 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829298.696349:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c3c0 x1874672085799552/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829298.696366:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498ed00 x1874672085798656/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829323 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829298.696426:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829298.696442:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829298.696447:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085798656:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829298.696456:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085798784:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787829298.696471:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829298.696482:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829298/real 1787829298] req@ffff9e35c498f840 x1874672085798784/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829323 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829298.696490:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f840 x1874672085798784/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829323 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829298.696496:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829298.696499:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829298.696501:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085798784:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829298.696529:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085798912:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829298.696541:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829298.696547:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829298/real 1787829298] req@ffff9e35c498cf00 x1874672085798912/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829323 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829298.696554:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498cf00 x1874672085798912/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829323 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829298.696560:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829298.696562:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829298.696564:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085798912:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829298.696568:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085799040:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829298.696578:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829298.696584:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829298/real 1787829298] req@ffff9e35c498d680 x1874672085799040/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829323 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829298.696592:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498d680 x1874672085799040/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829323 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829298.696611:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829298.696614:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829298.696617:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085799040:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829298.696624:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085799168:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787829298.696644:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829298.696657:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829298/real 1787829298] req@ffff9e35c498c780 x1874672085799168/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829323 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829298.696678:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c780 x1874672085799168/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829323 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829298.696684:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829298.696686:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829298.696688:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085799168:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829298.696693:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085799296:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787829298.696708:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829298.696715:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829298/real 1787829298] req@ffff9e35c498e940 x1874672085799296/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829323 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829298.696722:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e940 x1874672085799296/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829323 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829298.696733:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829298.696735:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829298.696737:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085799296:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829298.696741:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085799424:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829298.696751:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829298.696757:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829298/real 1787829298] req@ffff9e35c498fc00 x1874672085799424/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829323 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829298.696765:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498fc00 x1874672085799424/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829323 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829298.696771:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829298.696773:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829298.696774:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085799424:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829298.696778:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085799552:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829298.696789:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829298.696794:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829298/real 1787829298] req@ffff9e35c498c3c0 x1874672085799552/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829323 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829298.696830:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c3c0 x1874672085799552/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829323 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829298.696836:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829298.696838:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829298.696841:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085799552:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829298.823993:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829298.823995:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829298.824091:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829298.824093:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829299.847927:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829299.847927:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829299.847952:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829299.848060:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829300.871954:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829300.871990:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829300.872013:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829300.872041:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829301.895860:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829301.895906:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829301.895955:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829301.896045:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829302.919947:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829302.919950:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829302.919990:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829302.919992:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829303.815957:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829303.815969:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787829303.815975:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829303.816002:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2739 00000100:00080000:3.0:1787829303.816015:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829303.816024:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787829303.816065:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a580 x1874672085799680/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829303.816099:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829303.816104:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829303.816121:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829303.816127:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2739 00000100:00080000:3.0:1787829303.816131:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829303.816136:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829303.816164:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829303.816184:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b0c0 x1874672085799808/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829303.816201:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787829303.816212:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085799680:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1787829303.816212:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787829303.816224:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829303.816232:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2739 00000100:00080000:3.0:1787829303.816238:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829303.816244:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829303.816253:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829303.816271:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b480 x1874672085799936/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829303.816305:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829303.816309:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787829303.816311:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829303.816315:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2739 00000100:00080000:3.0:1787829303.816320:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1787829303.816328:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1787829303.816331:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829303.816339:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:3.0:1787829303.816357:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c64992c0 x1874672085800064/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829303.816368:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829303.816372:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787829303.816376:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829303.816383:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2739 00000100:00080000:3.0:1787829303.816388:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1787829303.816392:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829303/real 1787829303] req@ffff9e35c649a580 x1874672085799680/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829328 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829303.816402:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829303.816423:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:1.0:1787829303.816430:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a580 x1874672085799680/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829328 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829303.816443:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1787829303.816449:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499e00 x1874672085800192/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829303.816451:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829303.816462:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085799680:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1787829303.816486:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829303.816490:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829303.816492:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787829303.816501:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085799808:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1787829303.816501:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2739 00000100:00080000:3.0:1787829303.816516:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829303.816540:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:1.0:1787829303.816546:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:3.0:1787829303.816549:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829303.816559:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829303/real 1787829303] req@ffff9e35c649b0c0 x1874672085799808/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829328 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829303.816577:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b0c0 x1874672085799808/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829328 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829303.816579:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498000 x1874672085800320/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829303.816601:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829303.816612:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829303.816619:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085799808:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1787829303.816620:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829303.816628:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:1.0:1787829303.816635:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085799936:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1787829303.816636:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829303.816651:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2739 00000100:00080000:3.0:1787829303.816656:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829303.816660:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:1.0:1787829303.816667:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:3.0:1787829303.816667:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829303.816692:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829303/real 1787829303] req@ffff9e35c649b480 x1874672085799936/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829328 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829303.816702:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649ad00 x1874672085800448/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829303.816705:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b480 x1874672085799936/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829328 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829303.816763:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829303.816764:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829303.816779:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:3.0:1787829303.816780:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787829303.816785:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787829303.816792:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085799936:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1787829303.816797:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2739 00000100:00080000:3.0:1787829303.816801:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829303.816805:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829303.816810:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:3.0:1787829303.816835:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c64983c0 x1874672085800576/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829303.816859:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c64992c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085800064:192.168.203.141@tcp:8:kworker.0 00000100:00080000:3.0:1787829303.816861:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829303.816870:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000800:00000100:1.0:1787829303.816885:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829303.816895:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829303/real 1787829303] req@ffff9e35c64992c0 x1874672085800064/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829328 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829303.816904:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787829303.816905:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c64992c0 x1874672085800064/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829328 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829303.816921:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2739 00000100:00080000:1.0:1787829303.816944:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829303.816953:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829303.816954:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829303.816964:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c64992c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085800064:192.168.203.141@tcp:8:kworker.0 00000100:00080000:3.0:1787829303.816965:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829303.816980:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:1.0:1787829303.816983:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085800192:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787829303.817021:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a1c0 x1874672085800704/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787829303.817022:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829303.817038:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829303/real 1787829303] req@ffff9e35c6499e00 x1874672085800192/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829328 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829303.817049:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499e00 x1874672085800192/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829328 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829303.817061:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829303.817063:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829303.817065:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085800192:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829303.817083:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085800320:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787829303.817095:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829303.817102:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829303/real 1787829303] req@ffff9e35c6498000 x1874672085800320/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829328 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829303.817128:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498000 x1874672085800320/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829328 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829303.817134:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829303.817137:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829303.817139:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085800320:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829303.817144:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085800448:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787829303.817158:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829303.817166:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829303/real 1787829303] req@ffff9e35c649ad00 x1874672085800448/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829328 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829303.817179:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649ad00 x1874672085800448/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829328 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829303.817184:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829303.817186:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829303.817188:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085800448:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829303.817192:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c64983c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085800576:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829303.817202:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829303.817209:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829303/real 1787829303] req@ffff9e35c64983c0 x1874672085800576/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829328 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829303.817216:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c64983c0 x1874672085800576/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829328 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829303.817222:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829303.817224:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829303.817226:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c64983c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085800576:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829303.817230:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085800704:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829303.817239:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829303.817245:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829303/real 1787829303] req@ffff9e35c649a1c0 x1874672085800704/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829328 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829303.817253:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a1c0 x1874672085800704/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829328 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829303.817264:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829303.817267:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829303.817270:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085800704:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829303.943949:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829303.943949:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829303.943951:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829303.944001:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829304.967899:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829304.967902:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829304.967912:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829304.967941:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829305.991947:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829305.991948:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829305.991957:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829305.992002:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829307.015923:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829307.015923:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829307.015943:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829307.015976:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829308.039932:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829308.039932:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829308.039941:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829308.039975:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829308.935919:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829308.935935:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829308.935940:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829308.935954:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2744 00000100:00080000:2.0:1787829308.935968:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829308.935975:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787829308.936020:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e580 x1874672085800832/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829308.936060:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829308.936070:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829308.936083:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829308.936089:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2744 00000100:00080000:2.0:1787829308.936093:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829308.936097:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829308.936109:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829308.936124:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085800832:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1787829308.936126:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c000 x1874672085800960/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829308.936141:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829308.936144:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829308.936145:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829308.936148:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2744 00000100:00080000:2.0:1787829308.936151:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829308.936153:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829308.936156:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829308.936164:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cb40 x1874672085801088/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829308.936171:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829308.936174:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829308.936175:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829308.936178:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2744 00000100:00080000:2.0:1787829308.936180:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829308.936183:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829308.936185:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:2.0:1787829308.936194:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e1c0 x1874672085801216/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829308.936200:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829308.936203:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829308.936206:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1787829308.936208:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1787829308.936212:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2744 00000100:00080000:2.0:1787829308.936216:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829308.936219:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829308.936222:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:2.0:1787829308.936231:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f480 x1874672085801344/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829308.936237:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829308.936239:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829308.936241:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829308.936244:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2744 00000100:00080000:2.0:1787829308.936246:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829308.936249:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829308.936253:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829308.936264:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829308/real 1787829308] req@ffff9e35c498e580 x1874672085800832/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829333 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829308.936264:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f0c0 x1874672085801472/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829308.936289:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829308.936294:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829308.936296:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829308.936299:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2744 00000100:00080000:2.0:1787829308.936301:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829308.936307:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787829308.936308:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e580 x1874672085800832/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829333 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1787829308.936310:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829308.936322:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c3c0 x1874672085801600/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829308.936326:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829308.936334:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829308.936336:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:2.0:1787829308.936338:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829308.936342:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787829308.936346:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085800832:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1787829308.936348:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2744 00000100:00080000:2.0:1787829308.936353:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829308.936357:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829308.936360:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:2.0:1787829308.936367:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498fc00 x1874672085801728/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829308.936372:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829308.936375:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829308.936377:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787829308.936380:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085800960:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1787829308.936384:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2744 00000100:00080000:2.0:1787829308.936390:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829308.936392:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829308.936396:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:2.0:1787829308.936405:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e940 x1874672085801856/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787829308.936415:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829308.936430:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829308/real 1787829308] req@ffff9e35c498c000 x1874672085800960/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829333 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829308.936441:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c000 x1874672085800960/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829333 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829308.936448:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829308.936450:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829308.936453:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085800960:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829308.936457:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085801088:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787829308.936470:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829308.936476:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829308/real 1787829308] req@ffff9e35c498cb40 x1874672085801088/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829333 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829308.936484:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498cb40 x1874672085801088/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829333 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829308.936490:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829308.936491:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829308.936493:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085801088:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829308.936497:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085801216:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829308.936508:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829308.936514:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829308/real 1787829308] req@ffff9e35c498e1c0 x1874672085801216/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829333 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829308.936521:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e1c0 x1874672085801216/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829333 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829308.936527:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829308.936528:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829308.936531:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085801216:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829308.936534:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085801344:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829308.936544:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829308.936550:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829308/real 1787829308] req@ffff9e35c498f480 x1874672085801344/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829333 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829308.936557:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f480 x1874672085801344/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829333 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829308.936563:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829308.936564:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829308.936566:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085801344:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829308.936569:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085801472:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787829308.936579:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829308.936585:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829308/real 1787829308] req@ffff9e35c498f0c0 x1874672085801472/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829333 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829308.936593:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f0c0 x1874672085801472/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829333 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829308.936598:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829308.936599:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829308.936601:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085801472:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829308.936604:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085801600:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787829308.936614:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829308.936620:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829308/real 1787829308] req@ffff9e35c498c3c0 x1874672085801600/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829333 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829308.936632:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c3c0 x1874672085801600/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829333 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829308.936641:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829308.936644:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829308.936648:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085801600:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829308.936654:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085801728:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829308.936675:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829308.936688:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829308/real 1787829308] req@ffff9e35c498fc00 x1874672085801728/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829333 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829308.936701:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498fc00 x1874672085801728/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829333 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829308.936711:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829308.936714:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829308.936718:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085801728:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829308.936744:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085801856:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829308.936762:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829308.936772:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829308/real 1787829308] req@ffff9e35c498e940 x1874672085801856/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829333 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829308.936785:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e940 x1874672085801856/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829333 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829308.936794:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829308.936830:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829308.936835:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085801856:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829309.063899:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829309.063900:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829309.063962:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787829309.063992:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829310.087877:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829310.087881:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829310.087909:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787829310.087924:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829311.111927:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829311.111929:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829311.111976:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829311.111979:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829312.135931:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829312.135931:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829312.135931:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:1.0:1787829312.135993:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829313.159957:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829313.159957:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829313.160005:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787829313.160026:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829314.055869:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829314.055876:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829314.055880:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829314.055889:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2749 00000100:00080000:2.0:1787829314.055898:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829314.055904:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787829314.055937:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c780 x1874672085801984/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829314.055960:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829314.055964:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829314.055966:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829314.055971:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2749 00000100:00080000:2.0:1787829314.055974:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829314.055978:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829314.055989:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829314.056001:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498d680 x1874672085802112/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829314.056003:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085801984:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1787829314.056011:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829314.056015:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829314.056016:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829314.056020:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2749 00000100:00080000:2.0:1787829314.056024:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829314.056027:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829314.056032:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829314.056058:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cf00 x1874672085802240/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829314.056068:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829314.056070:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829314.056072:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829314.056076:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2749 00000100:00080000:2.0:1787829314.056079:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829314.056083:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:1.0:1787829314.056085:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:2.0:1787829314.056087:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:2.0:1787829314.056098:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f840 x1874672085802368/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829314.056106:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829314.056109:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829314.056110:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829314.056115:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2749 00000100:00080000:2.0:1787829314.056117:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829314.056121:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829314.056125:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:1.0:1787829314.056127:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829314/real 1787829314] req@ffff9e35c498c780 x1874672085801984/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829339 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829314.056141:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498ed00 x1874672085802496/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829314.056144:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c780 x1874672085801984/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829339 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829314.056155:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829314.056155:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829314.056159:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:2.0:1787829314.056159:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829314.056162:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787829314.056163:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085801984:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1787829314.056166:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2749 00000100:00080000:2.0:1787829314.056169:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829314.056172:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829314.056177:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829314.056186:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498de00 x1874672085802624/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829314.056188:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085802112:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1787829314.056196:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829314.056199:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829314.056200:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829314.056204:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2749 00000100:00080000:2.0:1787829314.056207:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1787829314.056210:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1787829314.056211:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829314.056215:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829314.056223:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829314/real 1787829314] req@ffff9e35c498d680 x1874672085802112/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829339 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829314.056225:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c780 x1874672085802752/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829314.056235:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787829314.056238:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498d680 x1874672085802112/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829339 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829314.056239:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829314.056241:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829314.056245:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2749 00000100:00080000:1.0:1787829314.056247:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829314.056248:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829314.056250:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:2.0:1787829314.056251:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787829314.056253:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085802112:192.168.203.141@tcp:38:kworker.0 00000008:00100000:2.0:1787829314.056256:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:1.0:1787829314.056259:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085802240:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829314.056266:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e940 x1874672085802880/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787829314.056275:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1787829314.056275:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829314.056279:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829314.056280:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829314.056284:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2749 00000100:00100000:1.0:1787829314.056286:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829314/real 1787829314] req@ffff9e35c498cf00 x1874672085802240/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829339 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829314.056287:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829314.056291:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829314.056295:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:1.0:1787829314.056298:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498cf00 x1874672085802240/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829339 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829314.056305:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498fc00 x1874672085803008/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829314.056307:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829314.056310:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829314.056314:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085802240:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829314.056319:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085802368:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829314.056338:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829314.056350:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829314/real 1787829314] req@ffff9e35c498f840 x1874672085802368/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829339 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829314.056360:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f840 x1874672085802368/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829339 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829314.056370:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829314.056372:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829314.056374:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085802368:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829314.056378:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085802496:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829314.056391:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829314.056398:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829314/real 1787829314] req@ffff9e35c498ed00 x1874672085802496/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829339 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829314.056405:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498ed00 x1874672085802496/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829339 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829314.056410:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829314.056412:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829314.056414:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085802496:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829314.056429:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085802624:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787829314.056440:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829314.056446:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829314/real 1787829314] req@ffff9e35c498de00 x1874672085802624/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829339 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829314.056453:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498de00 x1874672085802624/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829339 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829314.056458:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829314.056460:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829314.056463:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085802624:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829314.056467:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085802752:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787829314.056476:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829314.056489:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829314/real 1787829314] req@ffff9e35c498c780 x1874672085802752/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829339 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829314.056496:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c780 x1874672085802752/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829339 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829314.056502:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829314.056503:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829314.056505:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085802752:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829314.056509:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085802880:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829314.056519:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829314.056524:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829314/real 1787829314] req@ffff9e35c498e940 x1874672085802880/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829339 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829314.056531:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e940 x1874672085802880/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829339 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829314.056537:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829314.056538:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829314.056540:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085802880:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829314.056544:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085803008:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829314.056552:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829314.056558:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829314/real 1787829314] req@ffff9e35c498fc00 x1874672085803008/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829339 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829314.056566:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498fc00 x1874672085803008/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829339 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829314.056571:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829314.056572:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829314.056574:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085803008:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829314.183915:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829314.183941:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829314.183962:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829314.183964:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829315.207898:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829315.207898:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829315.207922:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829315.208004:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829316.231915:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829316.231915:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829316.231955:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:1.0:1787829316.231958:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829317.255866:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829317.255866:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829317.255877:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829317.255939:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829318.279892:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829318.279906:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829318.279906:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829318.279930:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829319.175950:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829319.175957:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829319.175961:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829319.175970:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2754 00000100:00080000:2.0:1787829319.175980:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829319.175985:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787829319.176031:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c3c0 x1874672085803136/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829319.176056:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829319.176059:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829319.176062:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829319.176066:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2754 00000100:00080000:2.0:1787829319.176069:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829319.176073:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829319.176083:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829319.176094:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f0c0 x1874672085803264/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829319.176104:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829319.176106:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829319.176108:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829319.176113:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2754 00000100:00100000:1.0:1787829319.176115:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085803136:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1787829319.176116:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829319.176122:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829319.176127:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829319.176136:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f480 x1874672085803392/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829319.176142:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829319.176145:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829319.176146:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829319.176150:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2754 00000100:00080000:2.0:1787829319.176158:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829319.176162:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829319.176166:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:2.0:1787829319.176175:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e1c0 x1874672085803520/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829319.176183:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829319.176187:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829319.176189:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829319.176194:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2754 00000100:00080000:2.0:1787829319.176199:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829319.176203:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829319.176209:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:1.0:1787829319.176224:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085803264:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829319.176226:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cb40 x1874672085803648/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829319.176243:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829319.176247:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829319.176250:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829319.176255:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2754 00000100:00080000:2.0:1787829319.176259:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1787829319.176262:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085803392:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1787829319.176265:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829319.176274:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829319.176286:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c000 x1874672085803776/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829319.176294:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787829319.176297:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085803520:192.168.203.141@tcp:8:kworker.0 00000100:00080000:2.0:1787829319.176298:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829319.176301:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829319.176308:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2754 00000100:00080000:2.0:1787829319.176313:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829319.176315:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829319.176319:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829319.176328:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e580 x1874672085803904/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829319.176336:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787829319.176338:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085803648:192.168.203.141@tcp:8:kworker.0 00000100:00080000:2.0:1787829319.176339:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829319.176343:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829319.176351:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2754 00000100:00080000:2.0:1787829319.176353:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829319.176355:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829319.176359:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:2.0:1787829319.176369:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498fc00 x1874672085804032/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829319.176374:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085803776:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1787829319.176386:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829319.176388:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829319.176390:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787829319.176399:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085803904:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1787829319.176403:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2754 00000100:00080000:2.0:1787829319.176408:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829319.176410:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829319.176414:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:1.0:1787829319.176420:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085804032:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829319.176429:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e940 x1874672085804160/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829319.176450:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085804160:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829319.303909:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829319.303911:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829319.303954:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787829319.303974:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829320.327922:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829320.327923:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829320.327963:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:1.0:1787829320.327981:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829321.351907:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829321.351929:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829321.351929:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829321.351972:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1787829322.248270:0:2397:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.141:988 00000400:00000100:2.0:1787829322.248345:0:2397:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.141@tcp at host 192.168.203.141 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1787829322.248372:0:2397:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:2.0:1787829322.248437:0:2397:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000800:00000100:2.0:1787829322.248455:0:2397:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:2.0:1787829322.248468:0:2397:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:2.0:1787829322.248479:0:2397:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:1.0:1787829322.248488:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829319/real 1787829322] req@ffff9e35c498e940 x1874672085804160/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829344 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:2.0:1787829322.248490:0:2397:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:2.0:1787829322.248506:0:2397:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:2.0:1787829322.248520:0:2397:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:1.0:1787829322.248533:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e940 x1874672085804160/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829344 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:2.0:1787829322.248542:0:2397:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00080000:1.0:1787829322.248560:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829322.248568:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829322.248575:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085804160:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829322.248584:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829319/real 1787829322] req@ffff9e35c498e580 x1874672085803904/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829344 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829322.248600:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e580 x1874672085803904/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829344 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829322.248609:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829322.248613:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829322.248617:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085803904:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829322.248625:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829319/real 1787829322] req@ffff9e35c498fc00 x1874672085804032/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829344 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829322.248645:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498fc00 x1874672085804032/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829344 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829322.248655:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829322.248659:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829322.248662:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085804032:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829322.248670:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829319/real 1787829322] req@ffff9e35c498cb40 x1874672085803648/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829344 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829322.248683:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498cb40 x1874672085803648/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829344 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829322.248692:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829322.248696:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829322.248700:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085803648:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829322.248706:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829319/real 1787829322] req@ffff9e35c498c000 x1874672085803776/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829344 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829322.248720:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c000 x1874672085803776/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829344 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829322.248730:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829322.248735:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829322.248739:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085803776:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829322.248747:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829319/real 1787829322] req@ffff9e35c498f0c0 x1874672085803264/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829344 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829322.248761:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f0c0 x1874672085803264/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829344 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829322.248777:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829322.248781:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829322.248784:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085803264:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829322.248791:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829319/real 1787829322] req@ffff9e35c498f480 x1874672085803392/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829344 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829322.248841:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f480 x1874672085803392/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829344 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829322.248850:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829322.248854:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829322.248857:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085803392:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829322.248864:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829319/real 1787829322] req@ffff9e35c498e1c0 x1874672085803520/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829344 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829322.248876:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e1c0 x1874672085803520/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829344 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829322.248885:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829322.248889:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829322.248892:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085803520:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829322.248899:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829319/real 1787829322] req@ffff9e35c498c3c0 x1874672085803136/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829344 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829322.248912:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c3c0 x1874672085803136/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829344 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829322.248919:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829322.248923:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829322.248926:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085803136:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1787829322.375926:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829322.375928:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829322.375947:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787829322.376051:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829323.399882:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829323.399882:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829323.399882:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787829323.399931:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829324.295902:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829324.295914:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829324.295918:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829324.295928:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2759 00000100:00080000:2.0:1787829324.295940:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829324.295948:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787829324.295999:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c780 x1874672085804288/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829324.296029:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829324.296033:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829324.296035:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829324.296039:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2759 00000100:00080000:2.0:1787829324.296043:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829324.296047:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829324.296058:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829324.296068:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085804288:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1787829324.296071:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498de00 x1874672085804416/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829324.296083:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829324.296085:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829324.296087:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829324.296089:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2759 00000100:00080000:2.0:1787829324.296092:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829324.296094:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829324.296097:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829324.296106:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498ed00 x1874672085804544/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829324.296113:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829324.296116:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829324.296118:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829324.296121:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2759 00000100:00080000:2.0:1787829324.296124:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829324.296127:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829324.296130:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:2.0:1787829324.296139:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f840 x1874672085804672/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829324.296148:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829324.296151:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829324.296152:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1787829324.296154:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1787829324.296157:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2759 00000100:00080000:2.0:1787829324.296161:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829324.296164:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829324.296168:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:2.0:1787829324.296176:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cf00 x1874672085804800/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829324.296183:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829324.296186:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829324.296187:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829324.296191:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2759 00000100:00080000:2.0:1787829324.296193:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829324.296196:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829324.296200:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829324.296205:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829324/real 1787829324] req@ffff9e35c498c780 x1874672085804288/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829349 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829324.296211:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498d680 x1874672085804928/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829324.296238:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829324.296241:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829324.296243:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787829324.296247:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c780 x1874672085804288/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829349 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829324.296248:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2759 00000100:00080000:2.0:1787829324.296253:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829324.296258:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829324.296267:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00080000:1.0:1787829324.296273:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:2.0:1787829324.296281:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c3c0 x1874672085805056/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829324.296282:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829324.296293:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085804288:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1787829324.296303:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829324.296314:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829324.296316:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829324.296323:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2759 00000100:00100000:1.0:1787829324.296328:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085804416:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1787829324.296328:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829324.296338:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829324.296344:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:2.0:1787829324.296362:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e1c0 x1874672085805184/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829324.296373:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1787829324.296374:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1787829324.296381:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829324.296383:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829324.296390:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2759 00000100:00080000:2.0:1787829324.296395:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1787829324.296397:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829324/real 1787829324] req@ffff9e35c498de00 x1874672085804416/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829349 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829324.296404:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829324.296414:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:1.0:1787829324.296423:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498de00 x1874672085804416/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829349 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829324.296432:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f480 x1874672085805312/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829324.296440:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829324.296449:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829324.296455:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085804416:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829324.296463:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085804544:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787829324.296489:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829324.296502:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829324/real 1787829324] req@ffff9e35c498ed00 x1874672085804544/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829349 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829324.296516:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498ed00 x1874672085804544/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829349 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829324.296524:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829324.296528:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829324.296531:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085804544:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829324.296537:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085804672:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829324.296560:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829324.296572:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829324/real 1787829324] req@ffff9e35c498f840 x1874672085804672/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829349 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829324.296583:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f840 x1874672085804672/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829349 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829324.296592:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829324.296596:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829324.296600:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085804672:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829324.296607:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085804800:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829324.296626:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829324.296638:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829324/real 1787829324] req@ffff9e35c498cf00 x1874672085804800/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829349 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829324.296649:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498cf00 x1874672085804800/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829349 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829324.296659:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829324.296662:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829324.296666:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085804800:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829324.296672:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085804928:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787829324.296694:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829324.296705:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829324/real 1787829324] req@ffff9e35c498d680 x1874672085804928/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829349 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829324.296717:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498d680 x1874672085804928/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829349 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829324.296726:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829324.296737:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829324.296741:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085804928:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829324.296747:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085805056:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787829324.296768:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829324.296778:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829324/real 1787829324] req@ffff9e35c498c3c0 x1874672085805056/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829349 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829324.296789:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c3c0 x1874672085805056/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829349 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829324.296822:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829324.296826:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829324.296830:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085805056:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829324.296858:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085805184:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829324.296880:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829324.296892:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829324/real 1787829324] req@ffff9e35c498e1c0 x1874672085805184/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829349 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829324.296904:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e1c0 x1874672085805184/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829349 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829324.296912:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829324.296915:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829324.296918:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085805184:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829324.296924:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085805312:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829324.296943:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829324.296956:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829324/real 1787829324] req@ffff9e35c498f480 x1874672085805312/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829349 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829324.296983:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f480 x1874672085805312/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829349 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829324.296992:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829324.296995:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829324.296998:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085805312:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829324.423894:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829324.423927:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829324.423943:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829324.423948:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829325.447907:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829325.447908:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829325.447913:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829325.447983:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829326.471897:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829326.471900:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787829326.471930:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829326.471943:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829327.495924:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829327.495924:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:3.0:1787829327.495924:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829327.496032:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829328.519894:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829328.519894:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829328.519925:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787829328.519957:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829329.415911:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829329.415922:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787829329.415937:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829329.415949:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2764 00000100:00080000:3.0:1787829329.415960:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829329.415968:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787829329.416015:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499680 x1874672085805440/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829329.416045:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829329.416048:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829329.416050:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829329.416054:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2764 00000100:00080000:3.0:1787829329.416057:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829329.416061:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829329.416072:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829329.416087:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085805440:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1787829329.416087:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498f00 x1874672085805568/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829329.416102:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829329.416104:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787829329.416105:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829329.416108:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2764 00000100:00080000:3.0:1787829329.416110:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829329.416112:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829329.416115:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829329.416124:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499a40 x1874672085805696/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829329.416135:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829329.416138:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787829329.416140:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829329.416143:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2764 00000100:00080000:3.0:1787829329.416147:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829329.416151:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829329.416157:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000800:00000100:1.0:1787829329.416168:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787829329.416172:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498780 x1874672085805824/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829329.416184:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829329.416189:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787829329.416192:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829329.416205:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2764 00000100:00080000:3.0:1787829329.416210:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829329.416215:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829329.416220:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:1.0:1787829329.416226:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829329/real 1787829329] req@ffff9e35c6499680 x1874672085805440/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829354 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829329.416237:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a940 x1874672085805952/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829329.416250:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787829329.416253:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499680 x1874672085805440/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829354 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829329.416256:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829329.416260:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829329.416273:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2764 00000100:00080000:1.0:1787829329.416278:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829329.416279:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829329.416286:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:3.0:1787829329.416289:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787829329.416294:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085805440:192.168.203.141@tcp:250:kworker.0 00000008:00100000:3.0:1787829329.416298:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829329.416315:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498b40 x1874672085806080/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829329.416324:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085805568:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1787829329.416330:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829329.416335:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787829329.416338:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829329.416345:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2764 00000100:00080000:3.0:1787829329.416349:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1787829329.416351:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1787829329.416356:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829329.416362:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829329.416370:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829329/real 1787829329] req@ffff9e35c6498f00 x1874672085805568/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829354 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829329.416379:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649bc00 x1874672085806208/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829329.416403:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787829329.416409:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498f00 x1874672085805568/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829354 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829329.416409:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787829329.416413:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829329.416424:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2764 00000100:00080000:1.0:1787829329.416427:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829329.416430:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829329.416433:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829329.416438:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085805568:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1787829329.416441:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787829329.416447:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085805696:192.168.203.141@tcp:38:kworker.0 00000008:00100000:3.0:1787829329.416448:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:3.0:1787829329.416463:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b840 x1874672085806336/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787829329.416474:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1787829329.416483:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829329.416486:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787829329.416488:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787829329.416489:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829329/real 1787829329] req@ffff9e35c6499a40 x1874672085805696/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829354 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829329.416495:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2764 00000100:00080000:3.0:1787829329.416504:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1787829329.416508:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499a40 x1874672085805696/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829354 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829329.416514:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:1.0:1787829329.416522:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000008:00100000:3.0:1787829329.416524:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00080000:1.0:1787829329.416526:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829329.416532:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085805696:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829329.416537:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085805824:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787829329.416554:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499680 x1874672085806464/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787829329.416555:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829329.416566:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829329/real 1787829329] req@ffff9e35c6498780 x1874672085805824/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829354 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829329.416578:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498780 x1874672085805824/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829354 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829329.416588:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829329.416592:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829329.416597:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085805824:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829329.416611:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085805952:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829329.416635:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829329.416644:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829329/real 1787829329] req@ffff9e35c649a940 x1874672085805952/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829354 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829329.416652:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a940 x1874672085805952/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829354 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829329.416659:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829329.416661:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829329.416663:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085805952:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829329.416684:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085806080:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787829329.416707:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829329.416719:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829329/real 1787829329] req@ffff9e35c6498b40 x1874672085806080/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829354 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829329.416732:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498b40 x1874672085806080/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829354 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829329.416741:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829329.416744:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829329.416748:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085806080:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829329.416755:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085806208:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787829329.416773:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829329.416782:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829329/real 1787829329] req@ffff9e35c649bc00 x1874672085806208/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829354 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829329.416790:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649bc00 x1874672085806208/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829354 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829329.416796:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829329.416826:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829329.416829:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085806208:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829329.416833:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085806336:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829329.416846:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829329.416856:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829329/real 1787829329] req@ffff9e35c649b840 x1874672085806336/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829354 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829329.416869:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b840 x1874672085806336/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829354 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829329.416878:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829329.416882:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829329.416885:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085806336:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829329.416892:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085806464:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829329.416913:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829329.416925:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829329/real 1787829329] req@ffff9e35c6499680 x1874672085806464/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829354 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829329.416953:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499680 x1874672085806464/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829354 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829329.416963:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829329.416967:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829329.416969:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085806464:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829329.543893:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829329.543896:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829329.543902:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787829329.543961:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829330.567908:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829330.567911:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829330.567911:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829330.568022:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829331.591926:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829331.591927:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829331.591967:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787829331.592036:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829332.615896:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829332.615945:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829332.615961:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787829332.616006:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829333.639940:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829333.639963:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829333.639968:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829333.639984:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829334.535924:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829334.535935:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829334.535941:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829334.535951:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2769 00000100:00080000:1.0:1787829334.535962:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829334.535970:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787829334.536012:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7840 x1874672085806592/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829334.536041:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829334.536044:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829334.536046:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829334.536050:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2769 00000100:00080000:1.0:1787829334.536052:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829334.536054:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829334.536064:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829334.536081:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5a40 x1874672085806720/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829334.536092:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829334.536095:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829334.536097:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787829334.536100:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085806592:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787829334.536102:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2769 00000100:00080000:1.0:1787829334.536110:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829334.536113:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829334.536116:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829334.536126:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5e00 x1874672085806848/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829334.536133:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829334.536136:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829334.536138:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829334.536141:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2769 00000100:00080000:1.0:1787829334.536143:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829334.536146:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829334.536149:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:1.0:1787829334.536157:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4b40 x1874672085806976/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829334.536165:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829334.536167:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829334.536169:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829334.536172:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2769 00000100:00080000:1.0:1787829334.536175:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1787829334.536178:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787829334.536182:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829334.536186:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:1.0:1787829334.536195:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f70c0 x1874672085807104/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829334.536202:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829334.536205:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829334.536207:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829334.536209:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2769 00000100:00080000:1.0:1787829334.536212:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829334.536214:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829334.536218:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829334.536226:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829334/real 1787829334] req@ffff9e35da6f7840 x1874672085806592/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829359 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829334.536230:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f43c0 x1874672085807232/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829334.536244:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787829334.536248:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7840 x1874672085806592/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829359 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829334.536249:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829334.536252:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829334.536260:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2769 00000100:00080000:1.0:1787829334.536265:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829334.536266:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829334.536270:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:2.0:1787829334.536275:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000008:00100000:1.0:1787829334.536276:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829334.536281:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085806592:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1787829334.536289:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f61c0 x1874672085807360/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829334.536302:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829334.536305:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:2.0:1787829334.536308:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085806720:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787829334.536309:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829334.536316:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2769 00000100:00080000:1.0:1787829334.536321:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829334.536323:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829334.536328:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000800:00000100:2.0:1787829334.536335:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829334.536343:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4f00 x1874672085807488/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829334.536348:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829334/real 1787829334] req@ffff9e35da6f5a40 x1874672085806720/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829359 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829334.536370:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5a40 x1874672085806720/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829359 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829334.536372:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829334.536383:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829334.536386:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829334.536388:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829334.536391:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:1.0:1787829334.536398:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2769 00000100:00100000:2.0:1787829334.536398:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085806720:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787829334.536407:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787829334.536409:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085806848:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787829334.536415:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829334.536421:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000800:00000100:2.0:1787829334.536427:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829334.536436:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6580 x1874672085807616/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829334.536439:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829334/real 1787829334] req@ffff9e35da6f5e00 x1874672085806848/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829359 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829334.536463:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5e00 x1874672085806848/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829359 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829334.536470:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829334.536473:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829334.536476:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085806848:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829334.536481:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085806976:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829334.536495:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829334.536509:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829334/real 1787829334] req@ffff9e35da6f4b40 x1874672085806976/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829359 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829334.536518:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4b40 x1874672085806976/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829359 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829334.536524:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829334.536526:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829334.536528:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085806976:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829334.536532:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085807104:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829334.536543:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829334.536550:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829334/real 1787829334] req@ffff9e35da6f70c0 x1874672085807104/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829359 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829334.536557:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f70c0 x1874672085807104/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829359 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829334.536562:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829334.536564:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829334.536566:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085807104:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829334.536570:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085807232:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787829334.536581:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829334.536587:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829334/real 1787829334] req@ffff9e35da6f43c0 x1874672085807232/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829359 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829334.536600:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f43c0 x1874672085807232/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829359 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829334.536605:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829334.536607:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829334.536609:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085807232:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829334.536612:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085807360:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787829334.536623:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829334.536629:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829334/real 1787829334] req@ffff9e35da6f61c0 x1874672085807360/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829359 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829334.536636:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f61c0 x1874672085807360/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829359 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829334.536641:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829334.536643:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829334.536645:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085807360:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829334.536663:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085807488:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829334.536673:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829334.536679:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829334/real 1787829334] req@ffff9e35da6f4f00 x1874672085807488/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829359 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829334.536687:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4f00 x1874672085807488/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829359 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829334.536697:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829334.536700:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829334.536702:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085807488:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829334.536705:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085807616:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829334.536714:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829334.536719:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829334/real 1787829334] req@ffff9e35da6f6580 x1874672085807616/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829359 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829334.536727:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6580 x1874672085807616/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829359 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829334.536732:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829334.536734:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829334.536736:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085807616:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829334.663917:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829334.663923:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829334.663957:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829334.663958:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829335.687926:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829335.687926:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829335.687964:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787829335.687979:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829336.711946:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829336.711946:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829336.711984:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829336.712011:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829337.735925:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829337.735964:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829337.735969:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829337.736001:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829338.759923:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829338.759958:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829338.759980:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829338.759998:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00080000:0.0:1787829339.655901:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829339.655929:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787829339.655933:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829339.655944:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2775 00000100:00080000:0.0:1787829339.655955:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829339.655963:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787829339.656012:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7192c0 x1874672085807744/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829339.656039:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829339.656044:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829339.656046:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829339.656051:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2775 00000100:00080000:0.0:1787829339.656055:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829339.656060:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829339.656074:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829339.656109:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718780 x1874672085807872/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829339.656121:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829339.656125:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787829339.656128:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787829339.656135:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085807744:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1787829339.656139:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2775 00000100:00080000:0.0:1787829339.656148:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829339.656156:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829339.656165:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829339.656181:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a1c0 x1874672085808000/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829339.656191:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829339.656194:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829339.656195:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829339.656200:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2775 00000100:00080000:0.0:1787829339.656207:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829339.656214:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:2.0:1787829339.656217:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:0.0:1787829339.656230:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:0.0:1787829339.656249:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7183c0 x1874672085808128/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829339.656259:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829339.656264:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829339.656268:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787829339.656268:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829339/real 1787829339] req@ffff9e35da7192c0 x1874672085807744/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829364 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829339.656294:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2775 00000100:00100000:2.0:1787829339.656302:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7192c0 x1874672085807744/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829364 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829339.656305:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829339.656316:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:2.0:1787829339.656323:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:0.0:1787829339.656326:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00080000:2.0:1787829339.656333:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829339.656341:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085807744:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1787829339.656349:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a940 x1874672085808256/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829339.656359:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829339.656362:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829339.656363:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829339.656368:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2775 00000100:00080000:0.0:1787829339.656373:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787829339.656374:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085807872:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787829339.656383:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829339.656391:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1787829339.656405:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829339.656411:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718f00 x1874672085808384/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829339.656419:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829339/real 1787829339] req@ffff9e35da718780 x1874672085807872/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829364 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829339.656459:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787829339.656464:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718780 x1874672085807872/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829364 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829339.656466:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787829339.656471:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829339.656489:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2775 00000100:00080000:2.0:1787829339.656492:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829339.656494:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829339.656498:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:0.0:1787829339.656504:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787829339.656504:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085807872:192.168.203.141@tcp:38:kworker.0 00000008:00100000:0.0:1787829339.656518:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829339.656523:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085808000:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829339.656537:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718000 x1874672085808512/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829339.656549:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787829339.656560:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787829339.656560:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829339/real 1787829339] req@ffff9e35da71a1c0 x1874672085808000/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829364 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829339.656577:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829339.656583:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829339.656602:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2775 00000100:00100000:2.0:1787829339.656606:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a1c0 x1874672085808000/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829364 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829339.656614:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829339.656616:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829339.656623:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:0.0:1787829339.656627:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787829339.656631:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085808000:192.168.203.141@tcp:38:kworker.0 00000008:00100000:0.0:1787829339.656640:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:2.0:1787829339.656641:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085808128:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829339.656663:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829339.656669:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a580 x1874672085808640/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829339.656678:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829339/real 1787829339] req@ffff9e35da7183c0 x1874672085808128/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829364 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829339.656712:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787829339.656718:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672085808128/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829364 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829339.656719:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829339.656723:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829339.656739:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2775 00000100:00080000:2.0:1787829339.656741:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829339.656745:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829339.656750:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:0.0:1787829339.656755:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787829339.656756:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085808128:192.168.203.141@tcp:8:kworker.0 00000008:00100000:0.0:1787829339.656770:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:2.0:1787829339.656772:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085808256:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829339.656846:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da719a40 x1874672085808768/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829339.656885:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829339.656914:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829339/real 1787829339] req@ffff9e35da71a940 x1874672085808256/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829364 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829339.656924:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672085808256/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829364 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829339.656935:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829339.656941:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829339.656944:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085808256:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829339.656968:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085808384:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787829339.656983:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829339.656990:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829339/real 1787829339] req@ffff9e35da718f00 x1874672085808384/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829364 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829339.656997:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672085808384/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829364 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829339.657003:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829339.657005:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829339.657007:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085808384:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829339.657010:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085808512:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787829339.657020:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829339.657026:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829339/real 1787829339] req@ffff9e35da718000 x1874672085808512/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829364 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829339.657033:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718000 x1874672085808512/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829364 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829339.657038:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829339.657040:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829339.657042:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085808512:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829339.657046:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085808640:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829339.657057:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829339.657076:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829339/real 1787829339] req@ffff9e35da71a580 x1874672085808640/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829364 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829339.657087:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672085808640/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829364 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829339.657096:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829339.657099:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829339.657102:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085808640:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829339.657109:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085808768:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829339.657127:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829339.657137:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829339/real 1787829339] req@ffff9e35da719a40 x1874672085808768/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829364 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829339.657149:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672085808768/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829364 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829339.657157:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829339.657161:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829339.657164:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085808768:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829339.783949:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829339.783971:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829339.783997:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787829339.784037:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829340.807877:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829340.807939:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787829340.807953:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829340.807965:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829341.831957:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829341.831957:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829341.832012:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:1.0:1787829341.832016:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829342.855956:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829342.855956:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829342.855997:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829342.856043:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829343.879931:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829343.879931:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829343.879953:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:1.0:1787829343.879986:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829344.775932:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829344.775943:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787829344.775948:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829344.775960:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2780 00000100:00080000:0.0:1787829344.775970:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829344.775977:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787829344.776018:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718b40 x1874672085808896/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829344.776052:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829344.776055:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829344.776057:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829344.776060:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2780 00000100:00080000:0.0:1787829344.776062:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829344.776065:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829344.776074:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829344.776087:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da719a40 x1874672085809024/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829344.776097:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829344.776101:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787829344.776105:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787829344.776107:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085808896:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1787829344.776115:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2780 00000100:00080000:0.0:1787829344.776125:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829344.776130:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829344.776137:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829344.776152:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a580 x1874672085809152/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829344.776170:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829344.776174:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000100:2.0:1787829344.776177:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787829344.776178:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829344.776185:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2780 00000100:00080000:0.0:1787829344.776190:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829344.776195:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829344.776200:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:0.0:1787829344.776210:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718000 x1874672085809280/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829344.776218:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829344.776221:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829344.776224:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787829344.776226:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829344/real 1787829344] req@ffff9e35da718b40 x1874672085808896/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829369 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829344.776230:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2780 00000100:00080000:0.0:1787829344.776236:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829344.776241:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829344.776245:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:2.0:1787829344.776253:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718b40 x1874672085808896/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829369 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829344.776256:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718f00 x1874672085809408/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829344.776281:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829344.776287:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829344.776292:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829344.776292:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:0.0:1787829344.776296:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787829344.776302:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085808896:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1787829344.776303:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2780 00000100:00080000:0.0:1787829344.776307:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829344.776314:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829344.776317:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829344.776325:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a940 x1874672085809536/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829344.776332:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829344.776334:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787829344.776336:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829344.776341:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2780 00000100:00100000:2.0:1787829344.776343:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085809024:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787829344.776344:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829344.776350:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829344.776356:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829344.776367:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7183c0 x1874672085809664/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829344.776374:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829344.776377:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829344.776379:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829344.776383:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2780 00000100:00080000:0.0:1787829344.776386:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1787829344.776388:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787829344.776392:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829344.776397:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:0.0:1787829344.776406:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a1c0 x1874672085809792/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829344.776412:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829344/real 1787829344] req@ffff9e35da719a40 x1874672085809024/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829369 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829344.776415:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829344.776421:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829344.776423:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829344.776432:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2780 00000100:00080000:0.0:1787829344.776437:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829344.776440:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829344.776443:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:2.0:1787829344.776446:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672085809024/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829369 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829344.776452:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718780 x1874672085809920/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829344.776471:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829344.776478:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829344.776484:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085809024:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829344.776490:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085809152:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787829344.776515:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829344.776528:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829344/real 1787829344] req@ffff9e35da71a580 x1874672085809152/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829369 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829344.776549:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672085809152/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829369 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829344.776559:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829344.776563:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829344.776567:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085809152:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829344.776574:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085809280:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829344.776596:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829344.776608:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829344/real 1787829344] req@ffff9e35da718000 x1874672085809280/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829369 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829344.776622:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718000 x1874672085809280/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829369 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829344.776630:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829344.776634:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829344.776638:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085809280:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829344.776644:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085809408:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829344.776665:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829344.776676:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829344/real 1787829344] req@ffff9e35da718f00 x1874672085809408/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829369 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829344.776696:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672085809408/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829369 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829344.776706:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829344.776708:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829344.776710:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085809408:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829344.776714:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085809536:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787829344.776728:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829344.776735:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829344/real 1787829344] req@ffff9e35da71a940 x1874672085809536/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829369 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829344.776743:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672085809536/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829369 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829344.776749:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829344.776751:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829344.776753:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085809536:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829344.776768:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085809664:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787829344.776779:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829344.776785:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829344/real 1787829344] req@ffff9e35da7183c0 x1874672085809664/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829369 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829344.776792:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672085809664/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829369 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829344.776839:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829344.776844:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829344.776847:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085809664:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829344.776854:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085809792:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829344.776876:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829344.776889:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829344/real 1787829344] req@ffff9e35da71a1c0 x1874672085809792/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829369 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829344.776902:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a1c0 x1874672085809792/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829369 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829344.776922:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829344.776925:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829344.776929:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085809792:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829344.776936:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085809920:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829344.776957:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829344.776970:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829344/real 1787829344] req@ffff9e35da718780 x1874672085809920/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829369 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829344.776990:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718780 x1874672085809920/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829369 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829344.776999:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829344.777003:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829344.777007:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085809920:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829344.903936:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829344.903936:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829344.903945:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829344.903979:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829345.927918:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829345.927919:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829345.927936:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829345.927973:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829346.951933:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829346.951933:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829346.951936:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787829346.952043:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829347.975934:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829347.975934:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829347.975969:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787829347.976007:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829348.999931:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829348.999963:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829348.999984:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787829349.000027:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829349.895960:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829349.895968:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829349.895971:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829349.895980:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2785 00000100:00080000:2.0:1787829349.895989:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829349.895995:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787829349.896030:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f0c0 x1874672085810048/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829349.896057:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829349.896059:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829349.896061:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829349.896064:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2785 00000100:00080000:2.0:1787829349.896066:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829349.896068:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829349.896077:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829349.896086:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c000 x1874672085810176/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829349.896093:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829349.896097:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829349.896098:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829349.896101:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2785 00000100:00080000:2.0:1787829349.896104:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829349.896106:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829349.896109:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829349.896115:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cb40 x1874672085810304/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829349.896122:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829349.896124:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829349.896125:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829349.896128:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2785 00000100:00080000:2.0:1787829349.896130:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1787829349.896140:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085810048:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1787829349.896141:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829349.896156:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:2.0:1787829349.896183:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498fc00 x1874672085810432/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829349.896195:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829349.896197:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829349.896200:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829349.896203:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2785 00000100:00080000:2.0:1787829349.896206:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829349.896210:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829349.896217:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:2.0:1787829349.896227:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e580 x1874672085810560/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787829349.896229:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1787829349.896254:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829349.896258:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829349.896260:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829349.896267:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2785 00000100:00080000:2.0:1787829349.896271:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1787829349.896277:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829349/real 1787829349] req@ffff9e35c498f0c0 x1874672085810048/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829374 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829349.896282:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829349.896301:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829349.896313:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f0c0 x1874672085810048/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829374 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829349.896323:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e940 x1874672085810688/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829349.896341:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829349.896345:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829349.896351:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829349.896352:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:2.0:1787829349.896354:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787829349.896365:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085810048:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1787829349.896366:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2785 00000100:00080000:2.0:1787829349.896376:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829349.896383:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787829349.896384:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085810176:192.168.203.141@tcp:38:kworker.0 00000008:00100000:2.0:1787829349.896401:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829349.896417:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f480 x1874672085810816/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787829349.896424:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1787829349.896438:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787829349.896442:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829349/real 1787829349] req@ffff9e35c498c000 x1874672085810176/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829374 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829349.896443:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829349.896450:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829349.896467:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2785 00000100:00080000:2.0:1787829349.896477:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829349.896491:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787829349.896494:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c000 x1874672085810176/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829374 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1787829349.896501:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00080000:1.0:1787829349.896526:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:2.0:1787829349.896531:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e1c0 x1874672085810944/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829349.896536:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829349.896551:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085810176:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1787829349.896565:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829349.896568:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829349.896570:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829349.896575:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2785 00000100:00080000:2.0:1787829349.896579:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829349.896583:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829349.896586:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:1.0:1787829349.896597:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085810304:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829349.896602:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c3c0 x1874672085811072/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787829349.896629:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829349.896641:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829349/real 1787829349] req@ffff9e35c498cb40 x1874672085810304/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829374 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829349.896653:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498cb40 x1874672085810304/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829374 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829349.896662:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829349.896667:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829349.896670:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085810304:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829349.896678:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085810432:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829349.896701:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829349.896714:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829349/real 1787829349] req@ffff9e35c498fc00 x1874672085810432/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829374 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829349.896727:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498fc00 x1874672085810432/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829374 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829349.896742:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829349.896745:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829349.896747:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085810432:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829349.896752:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085810560:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829349.896769:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829349.896776:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829349/real 1787829349] req@ffff9e35c498e580 x1874672085810560/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829374 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829349.896784:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e580 x1874672085810560/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829374 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829349.896834:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829349.896838:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829349.896841:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085810560:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829349.896849:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085810688:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787829349.896874:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829349.896888:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829349/real 1787829349] req@ffff9e35c498e940 x1874672085810688/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829374 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829349.896902:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e940 x1874672085810688/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829374 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829349.896911:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829349.896915:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829349.896919:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085810688:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829349.896927:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085810816:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787829349.896949:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829349.896958:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829349/real 1787829349] req@ffff9e35c498f480 x1874672085810816/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829374 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829349.896966:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f480 x1874672085810816/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829374 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829349.896972:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829349.896974:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829349.896976:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085810816:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829349.896980:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085810944:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829349.896992:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829349.896998:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829349/real 1787829349] req@ffff9e35c498e1c0 x1874672085810944/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829374 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829349.897004:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e1c0 x1874672085810944/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829374 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829349.897010:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829349.897013:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829349.897014:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085810944:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829349.897032:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085811072:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829349.897043:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829349.897050:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829349/real 1787829349] req@ffff9e35c498c3c0 x1874672085811072/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829374 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829349.897061:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c3c0 x1874672085811072/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829374 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829349.897071:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829349.897075:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829349.897079:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085811072:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829350.023902:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829350.023932:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829350.023935:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787829350.023998:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829351.047894:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829351.047894:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829351.047894:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829351.047952:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829352.071907:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829352.071953:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829352.071954:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787829352.072005:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829353.095915:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829353.095937:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829353.095966:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:1.0:1787829353.096010:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829354.119912:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829354.119950:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829354.119958:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829354.119976:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829355.015913:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829355.015925:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829355.015930:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829355.015947:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2790 00000100:00080000:2.0:1787829355.015959:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829355.015967:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787829355.016009:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498d680 x1874672085811200/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829355.016036:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829355.016040:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829355.016041:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829355.016044:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2790 00000100:00080000:2.0:1787829355.016048:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829355.016051:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829355.016062:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829355.016076:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cf00 x1874672085811328/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829355.016086:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829355.016089:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829355.016091:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829355.016095:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2790 00000100:00080000:2.0:1787829355.016098:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1787829355.016108:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085811200:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1787829355.016108:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829355.016115:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829355.016129:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f840 x1874672085811456/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829355.016139:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829355.016142:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829355.016145:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829355.016150:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2790 00000100:00080000:2.0:1787829355.016156:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829355.016161:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829355.016167:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:2.0:1787829355.016181:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498ed00 x1874672085811584/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829355.016193:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1787829355.016199:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1787829355.016201:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829355.016206:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829355.016218:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2790 00000100:00080000:2.0:1787829355.016224:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829355.016229:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829355.016233:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:2.0:1787829355.016248:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498de00 x1874672085811712/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829355.016255:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829355.016259:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:1.0:1787829355.016261:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829355/real 1787829355] req@ffff9e35c498d680 x1874672085811200/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829380 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829355.016262:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829355.016268:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2790 00000100:00080000:2.0:1787829355.016277:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829355.016281:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829355.016291:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829355.016309:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c780 x1874672085811840/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829355.016320:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829355.016324:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:1.0:1787829355.016328:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498d680 x1874672085811200/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829380 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829355.016332:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829355.016347:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2790 00000100:00080000:2.0:1787829355.016356:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829355.016361:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:1.0:1787829355.016363:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:2.0:1787829355.016366:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00080000:1.0:1787829355.016378:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829355.016389:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c3c0 x1874672085811968/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829355.016392:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085811200:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1787829355.016422:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829355.016426:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:1.0:1787829355.016430:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085811328:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1787829355.016431:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829355.016445:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2790 00000100:00080000:2.0:1787829355.016450:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829355.016456:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829355.016463:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000800:00000100:1.0:1787829355.016478:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829355.016488:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e1c0 x1874672085812096/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829355.016499:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829355.016504:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:1.0:1787829355.016505:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829355/real 1787829355] req@ffff9e35c498cf00 x1874672085811328/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829380 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829355.016508:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829355.016518:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2790 00000100:00080000:2.0:1787829355.016526:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829355.016535:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829355.016542:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:1.0:1787829355.016549:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498cf00 x1874672085811328/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829380 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829355.016554:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f480 x1874672085812224/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829355.016604:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829355.016613:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829355.016619:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085811328:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829355.016652:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085811456:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787829355.016680:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829355.016694:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829355/real 1787829355] req@ffff9e35c498f840 x1874672085811456/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829380 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829355.016707:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f840 x1874672085811456/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829380 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829355.016716:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829355.016719:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829355.016723:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085811456:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829355.016730:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085811584:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829355.016770:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829355.016783:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829355/real 1787829355] req@ffff9e35c498ed00 x1874672085811584/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829380 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829355.016847:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498ed00 x1874672085811584/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829380 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829355.016864:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829355.016868:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829355.016873:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085811584:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829355.016881:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085811712:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829355.016904:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829355.016918:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829355/real 1787829355] req@ffff9e35c498de00 x1874672085811712/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829380 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829355.016929:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498de00 x1874672085811712/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829380 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829355.016938:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829355.016942:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829355.016945:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085811712:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829355.016951:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085811840:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787829355.016971:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829355.016983:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829355/real 1787829355] req@ffff9e35c498c780 x1874672085811840/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829380 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829355.016994:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c780 x1874672085811840/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829380 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829355.017008:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829355.017012:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829355.017015:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085811840:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829355.017022:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085811968:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787829355.017044:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829355.017058:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829355/real 1787829355] req@ffff9e35c498c3c0 x1874672085811968/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829380 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829355.017070:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c3c0 x1874672085811968/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829380 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829355.017078:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829355.017081:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829355.017085:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085811968:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829355.017091:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085812096:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829355.017112:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829355.017124:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829355/real 1787829355] req@ffff9e35c498e1c0 x1874672085812096/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829380 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829355.017137:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e1c0 x1874672085812096/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829380 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829355.017145:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829355.017149:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829355.017152:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085812096:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829355.017159:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085812224:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829355.017180:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829355.017192:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829355/real 1787829355] req@ffff9e35c498f480 x1874672085812224/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829380 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829355.017204:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f480 x1874672085812224/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829380 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829355.017213:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829355.017217:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829355.017220:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085812224:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829355.143951:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829355.143951:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829355.143952:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787829355.144021:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829356.167889:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829356.167889:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829356.167893:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787829356.167944:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829357.191861:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829357.191886:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829357.191902:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787829357.191908:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829358.215863:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829358.215871:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829358.215876:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787829358.215954:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829359.239895:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829359.239916:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829359.239932:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829359.239935:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829360.135883:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829360.135893:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829360.135897:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829360.135908:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2795 00000100:00080000:1.0:1787829360.135927:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829360.135933:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787829360.135973:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4780 x1874672085812352/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829360.136017:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829360.136026:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829360.136029:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829360.136035:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2795 00000100:00080000:1.0:1787829360.136039:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829360.136045:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787829360.136053:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085812352:192.168.203.141@tcp:250:kworker.0 00000008:00100000:1.0:1787829360.136056:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829360.136074:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7480 x1874672085812480/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829360.136087:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829360.136091:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829360.136093:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829360.136098:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2795 00000100:00080000:1.0:1787829360.136103:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829360.136107:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829360.136113:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1787829360.136120:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829360.136130:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4000 x1874672085812608/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829360.136142:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829360.136146:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829360.136148:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829360.136153:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2795 00000100:00080000:1.0:1787829360.136158:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787829360.136159:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829360/real 1787829360] req@ffff9e35da6f4780 x1874672085812352/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829385 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829360.136168:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829360.136179:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:2.0:1787829360.136192:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4780 x1874672085812352/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829385 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829360.136197:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6940 x1874672085812736/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829360.136217:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829360.136222:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829360.136224:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829360.136231:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085812352:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787829360.136232:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829360.136237:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829360.136244:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2795 00000100:00080000:1.0:1787829360.136250:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829360.136254:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829360.136261:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:2.0:1787829360.136268:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085812480:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829360.136278:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6d00 x1874672085812864/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829360.136292:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1787829360.136296:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787829360.136300:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829360.136304:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829360.136311:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2795 00000100:00100000:2.0:1787829360.136313:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829360/real 1787829360] req@ffff9e35da6f7480 x1874672085812480/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829385 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829360.136319:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829360.136329:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787829360.136329:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7480 x1874672085812480/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829385 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1787829360.136339:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1787829360.136341:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829360.136347:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829360.136349:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085812480:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829360.136354:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085812608:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829360.136360:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4780 x1874672085812992/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829360.136370:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787829360.136373:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829360.136379:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:2.0:1787829360.136380:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829360/real 1787829360] req@ffff9e35da6f4000 x1874672085812608/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829385 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829360.136384:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829360.136395:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2795 00000100:00100000:2.0:1787829360.136396:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4000 x1874672085812608/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829385 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829360.136403:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829360.136406:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829360.136412:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:2.0:1787829360.136412:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829360.136420:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085812608:192.168.203.141@tcp:38:kworker.0 00000008:00100000:1.0:1787829360.136423:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829360.136426:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085812736:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829360.136441:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6580 x1874672085813120/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829360.136451:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829360.136460:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829360/real 1787829360] req@ffff9e35da6f6940 x1874672085812736/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829385 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829360.136463:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829360.136473:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829360.136477:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787829360.136478:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6940 x1874672085812736/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829385 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829360.136487:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2795 00000100:00080000:2.0:1787829360.136491:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829360.136493:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829360.136497:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:1.0:1787829360.136503:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787829360.136504:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085812736:192.168.203.141@tcp:8:kworker.0 00000008:00100000:1.0:1787829360.136519:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:2.0:1787829360.136525:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085812864:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829360.136534:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4f00 x1874672085813248/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829360.136543:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787829360.136544:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829360.136548:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829360.136550:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829360.136555:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2795 00000100:00100000:2.0:1787829360.136555:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829360/real 1787829360] req@ffff9e35da6f6d00 x1874672085812864/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829385 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829360.136559:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829360.136562:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829360.136569:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:2.0:1787829360.136580:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6d00 x1874672085812864/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829385 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829360.136584:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f61c0 x1874672085813376/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829360.136612:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829360.136617:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829360.136620:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085812864:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829360.136626:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085812992:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787829360.136639:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829360.136646:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829360/real 1787829360] req@ffff9e35da6f4780 x1874672085812992/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829385 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829360.136653:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4780 x1874672085812992/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829385 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829360.136663:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829360.136665:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829360.136667:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085812992:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829360.136670:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085813120:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787829360.136680:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829360.136685:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829360/real 1787829360] req@ffff9e35da6f6580 x1874672085813120/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829385 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829360.136692:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6580 x1874672085813120/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829385 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829360.136697:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829360.136699:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829360.136701:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085813120:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829360.136711:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085813248:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829360.136739:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829360.136747:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829360/real 1787829360] req@ffff9e35da6f4f00 x1874672085813248/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829385 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829360.136754:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4f00 x1874672085813248/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829385 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829360.136770:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829360.136772:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829360.136775:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085813248:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829360.136778:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085813376:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829360.136789:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829360.136795:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829360/real 1787829360] req@ffff9e35da6f61c0 x1874672085813376/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829385 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829360.136820:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f61c0 x1874672085813376/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829385 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829360.136826:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829360.136828:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829360.136830:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085813376:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829360.263858:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829360.263874:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829360.263905:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787829360.263930:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829361.287927:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829361.287935:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829361.287955:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829361.287969:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829362.311915:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829362.311915:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829362.311915:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787829362.311966:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829363.335944:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829363.335988:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829363.336001:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829363.336038:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829364.359946:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829364.359958:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829364.359983:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787829364.360022:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829365.255946:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829365.255961:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787829365.255966:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829365.255980:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2800 00000100:00080000:3.0:1787829365.255995:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829365.256005:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787829365.256047:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a1c0 x1874672085813504/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829365.256081:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829365.256086:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829365.256088:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829365.256093:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2800 00000100:00080000:3.0:1787829365.256097:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829365.256102:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829365.256120:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829365.256135:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085813504:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1787829365.256140:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c64983c0 x1874672085813632/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829365.256155:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829365.256159:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787829365.256162:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829365.256166:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2800 00000100:00080000:3.0:1787829365.256172:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829365.256176:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829365.256181:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829365.256199:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649ad00 x1874672085813760/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829365.256208:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1787829365.256218:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829365.256221:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787829365.256223:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829365.256227:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2800 00000100:00080000:3.0:1787829365.256235:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829365.256238:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829365.256241:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:3.0:1787829365.256251:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498000 x1874672085813888/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829365.256258:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829365.256260:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787829365.256261:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787829365.256268:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829365/real 1787829365] req@ffff9e35c649a1c0 x1874672085813504/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829390 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829365.256268:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2800 00000100:00080000:3.0:1787829365.256270:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829365.256276:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829365.256283:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:3.0:1787829365.256295:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499e00 x1874672085814016/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829365.256300:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a1c0 x1874672085813504/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829390 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829365.256308:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829365.256315:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829365.256317:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829365.256324:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2800 00000100:00080000:2.0:1787829365.256327:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829365.256328:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829365.256335:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:2.0:1787829365.256340:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000008:00100000:3.0:1787829365.256342:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829365.256350:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085813504:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1787829365.256354:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c64992c0 x1874672085814144/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829365.256364:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829365.256367:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787829365.256369:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829365.256372:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2800 00000100:00080000:3.0:1787829365.256375:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829365.256377:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829365.256379:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829365.256387:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c64983c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085813632:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787829365.256390:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b480 x1874672085814272/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829365.256403:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829365.256408:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787829365.256410:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1787829365.256415:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1787829365.256418:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2800 00000100:00080000:3.0:1787829365.256425:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787829365.256431:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829365/real 1787829365] req@ffff9e35c64983c0 x1874672085813632/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829390 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829365.256433:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829365.256444:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:2.0:1787829365.256449:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c64983c0 x1874672085813632/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829390 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829365.256458:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1787829365.256462:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b0c0 x1874672085814400/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829365.256465:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829365.256471:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c64983c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085813632:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829365.256481:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085813760:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1787829365.256497:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1787829365.256502:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1787829365.256504:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:2.0:1787829365.256512:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829365/real 1787829365] req@ffff9e35c649ad00 x1874672085813760/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829390 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829365.256515:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829365.256526:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2800 00000100:00100000:2.0:1787829365.256530:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649ad00 x1874672085813760/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829390 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829365.256533:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829365.256542:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:2.0:1787829365.256543:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000008:00100000:3.0:1787829365.256548:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00080000:2.0:1787829365.256549:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829365.256555:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085813760:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829365.256562:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085813888:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787829365.256567:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a580 x1874672085814528/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829365.256583:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829365.256592:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829365/real 1787829365] req@ffff9e35c6498000 x1874672085813888/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829390 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829365.256601:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498000 x1874672085813888/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829390 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829365.256608:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829365.256610:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829365.256613:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085813888:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829365.256616:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085814016:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829365.256627:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829365.256633:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829365/real 1787829365] req@ffff9e35c6499e00 x1874672085814016/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829390 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829365.256641:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499e00 x1874672085814016/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829390 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829365.256647:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829365.256649:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829365.256651:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085814016:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829365.256655:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c64992c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085814144:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787829365.256666:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829365.256672:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829365/real 1787829365] req@ffff9e35c64992c0 x1874672085814144/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829390 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829365.256694:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c64992c0 x1874672085814144/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829390 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829365.256701:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829365.256704:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829365.256706:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c64992c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085814144:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829365.256723:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085814272:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787829365.256738:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829365.256745:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829365/real 1787829365] req@ffff9e35c649b480 x1874672085814272/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829390 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829365.256753:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b480 x1874672085814272/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829390 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829365.256759:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829365.256761:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829365.256763:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085814272:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829365.256767:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085814400:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829365.256782:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829365.256788:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829365/real 1787829365] req@ffff9e35c649b0c0 x1874672085814400/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829390 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829365.256795:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b0c0 x1874672085814400/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829390 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829365.256831:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829365.256834:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829365.256837:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085814400:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829365.256841:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085814528:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829365.256854:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829365.256861:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829365/real 1787829365] req@ffff9e35c649a580 x1874672085814528/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829390 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829365.256869:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a580 x1874672085814528/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829390 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829365.256875:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829365.256878:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829365.256882:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085814528:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829365.383954:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829365.383961:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829365.383991:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829365.384025:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829366.407879:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829366.407908:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829366.407927:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829366.407962:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829367.431884:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829367.431898:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829367.431922:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787829367.431964:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829368.455940:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829368.455940:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829368.456069:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829368.456113:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829369.479888:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829369.479888:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829369.479915:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829369.479967:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829370.375877:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829370.375885:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787829370.375889:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829370.375898:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2805 00000100:00080000:3.0:1787829370.375908:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829370.375914:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787829370.375947:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a580 x1874672085814656/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829370.375970:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829370.375973:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829370.375975:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829370.375978:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2805 00000100:00080000:3.0:1787829370.375980:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829370.375983:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829370.375992:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829370.376002:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b0c0 x1874672085814784/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829370.376013:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829370.376017:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787829370.376019:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829370.376024:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2805 00000100:00080000:3.0:1787829370.376028:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829370.376033:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829370.376039:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829370.376047:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085814656:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1787829370.376054:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b480 x1874672085814912/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829370.376064:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829370.376066:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787829370.376068:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829370.376071:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2805 00000100:00080000:3.0:1787829370.376074:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829370.376077:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829370.376081:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:3.0:1787829370.376091:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c64992c0 x1874672085815040/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829370.376097:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829370.376100:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787829370.376101:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829370.376104:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2805 00000100:00080000:3.0:1787829370.376106:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829370.376111:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:2.0:1787829370.376112:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:3.0:1787829370.376115:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:3.0:1787829370.376125:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499e00 x1874672085815168/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829370.376133:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829370.376135:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829370.376136:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829370.376139:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2805 00000100:00080000:3.0:1787829370.376141:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829370.376143:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829370.376147:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829370.376155:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829370/real 1787829370] req@ffff9e35c649a580 x1874672085814656/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829395 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829370.376160:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498000 x1874672085815296/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829370.376178:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787829370.376181:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a580 x1874672085814656/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829395 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829370.376184:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787829370.376189:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829370.376195:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829370.376198:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2805 00000100:00080000:2.0:1787829370.376202:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829370.376206:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085814656:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1787829370.376206:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829370.376215:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787829370.376217:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085814784:192.168.203.141@tcp:38:kworker.0 00000008:00100000:3.0:1787829370.376222:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829370.376237:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649ad00 x1874672085815424/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829370.376243:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1787829370.376252:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787829370.376256:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829370/real 1787829370] req@ffff9e35c649b0c0 x1874672085814784/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829395 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829370.376257:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787829370.376261:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829370.376269:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2805 00000100:00100000:2.0:1787829370.376275:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b0c0 x1874672085814784/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829395 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829370.376282:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829370.376283:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829370.376290:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829370.376297:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00080000:2.0:1787829370.376302:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829370.376306:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085814784:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787829370.376314:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c64983c0 x1874672085815552/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829370.376325:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829370.376328:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787829370.376330:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787829370.376334:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085814912:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1787829370.376337:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2805 00000100:00080000:3.0:1787829370.376343:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829370.376346:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829370.376350:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000800:00000100:2.0:1787829370.376356:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829370.376366:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829370/real 1787829370] req@ffff9e35c649b480 x1874672085814912/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829395 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829370.376371:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a1c0 x1874672085815680/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829370.376398:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b480 x1874672085814912/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829395 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829370.376409:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829370.376413:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829370.376418:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085814912:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829370.376426:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c64992c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085815040:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829370.376446:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829370.376455:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829370/real 1787829370] req@ffff9e35c64992c0 x1874672085815040/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829395 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829370.376462:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c64992c0 x1874672085815040/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829395 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829370.376468:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829370.376471:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829370.376474:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c64992c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085815040:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829370.376481:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085815168:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829370.376500:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829370.376512:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829370/real 1787829370] req@ffff9e35c6499e00 x1874672085815168/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829395 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829370.376525:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499e00 x1874672085815168/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829395 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829370.376534:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829370.376536:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829370.376538:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085815168:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829370.376543:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085815296:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787829370.376558:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829370.376565:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829370/real 1787829370] req@ffff9e35c6498000 x1874672085815296/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829395 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829370.376577:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498000 x1874672085815296/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829395 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829370.376583:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829370.376584:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829370.376586:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085815296:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829370.376589:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085815424:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787829370.376600:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829370.376606:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829370/real 1787829370] req@ffff9e35c649ad00 x1874672085815424/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829395 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829370.376613:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649ad00 x1874672085815424/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829395 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829370.376618:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829370.376620:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829370.376621:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085815424:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829370.376625:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c64983c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085815552:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829370.376640:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829370.376667:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829370/real 1787829370] req@ffff9e35c64983c0 x1874672085815552/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829395 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829370.376680:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c64983c0 x1874672085815552/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829395 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829370.376696:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829370.376699:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829370.376702:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c64983c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085815552:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829370.376720:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085815680:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829370.376736:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829370.376744:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829370/real 1787829370] req@ffff9e35c649a1c0 x1874672085815680/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829395 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829370.376751:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a1c0 x1874672085815680/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829395 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829370.376757:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829370.376759:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829370.376761:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085815680:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829370.503939:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829370.503946:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829370.503957:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829370.504008:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829371.527891:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829371.527930:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829371.527965:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829371.527971:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829372.551938:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829372.551955:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829372.551984:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829372.552050:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829373.575909:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829373.575910:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829373.575910:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829373.575946:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829374.599903:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829374.599914:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829374.599945:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829374.599947:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829375.495929:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829375.495940:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829375.495945:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829375.495956:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2810 00000100:00080000:2.0:1787829375.495967:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829375.495974:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787829375.496017:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e940 x1874672085815808/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829375.496046:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829375.496049:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829375.496051:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829375.496054:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2810 00000100:00080000:2.0:1787829375.496057:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829375.496059:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829375.496071:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829375.496086:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e580 x1874672085815936/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829375.496095:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085815808:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1787829375.496103:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829375.496109:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829375.496111:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829375.496117:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2810 00000100:00080000:2.0:1787829375.496121:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829375.496125:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829375.496129:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829375.496141:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498fc00 x1874672085816064/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829375.496149:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829375.496159:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829375.496161:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829375.496165:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2810 00000100:00080000:2.0:1787829375.496168:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829375.496173:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:1.0:1787829375.496174:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:2.0:1787829375.496180:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:2.0:1787829375.496191:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cb40 x1874672085816192/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829375.496201:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829375.496204:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829375.496206:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829375.496210:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2810 00000100:00080000:2.0:1787829375.496215:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1787829375.496219:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829375/real 1787829375] req@ffff9e35c498e940 x1874672085815808/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829400 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829375.496223:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829375.496236:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:1.0:1787829375.496242:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e940 x1874672085815808/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829400 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829375.496254:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:2.0:1787829375.496255:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c000 x1874672085816320/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829375.496260:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829375.496267:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085815808:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1787829375.496277:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829375.496280:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829375.496282:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829375.496287:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2810 00000100:00080000:2.0:1787829375.496290:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829375.496294:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787829375.496298:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085815936:192.168.203.141@tcp:38:kworker.0 00000008:00100000:2.0:1787829375.496300:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829375.496315:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f0c0 x1874672085816448/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787829375.496324:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1787829375.496327:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829375.496331:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829375.496333:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829375.496338:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2810 00000100:00100000:1.0:1787829375.496340:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829375/real 1787829375] req@ffff9e35c498e580 x1874672085815936/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829400 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829375.496342:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829375.496350:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829375.496357:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829375.496363:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e580 x1874672085815936/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829400 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829375.496369:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e940 x1874672085816576/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829375.496377:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829375.496382:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829375.496384:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:2.0:1787829375.496387:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829375.496389:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787829375.496390:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085815936:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1787829375.496395:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2810 00000100:00080000:2.0:1787829375.496398:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1787829375.496402:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085816064:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1787829375.496404:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829375.496411:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:2.0:1787829375.496428:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f480 x1874672085816704/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787829375.496438:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1787829375.496447:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829375.496452:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829375.496454:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787829375.496457:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829375/real 1787829375] req@ffff9e35c498fc00 x1874672085816064/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829400 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829375.496464:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2810 00000100:00080000:2.0:1787829375.496472:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829375.496478:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787829375.496481:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498fc00 x1874672085816064/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829400 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1787829375.496484:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00080000:1.0:1787829375.496493:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829375.496497:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829375.496502:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085816064:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829375.496502:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e1c0 x1874672085816832/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829375.496514:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085816192:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829375.496538:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829375.496547:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829375/real 1787829375] req@ffff9e35c498cb40 x1874672085816192/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829400 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829375.496555:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498cb40 x1874672085816192/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829400 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829375.496561:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829375.496563:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829375.496565:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085816192:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829375.496568:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085816320:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829375.496579:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829375.496585:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829375/real 1787829375] req@ffff9e35c498c000 x1874672085816320/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829400 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829375.496592:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c000 x1874672085816320/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829400 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829375.496599:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829375.496600:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829375.496602:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085816320:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829375.496635:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085816448:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787829375.496651:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829375.496658:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829375/real 1787829375] req@ffff9e35c498f0c0 x1874672085816448/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829400 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829375.496666:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f0c0 x1874672085816448/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829400 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829375.496673:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829375.496675:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829375.496677:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085816448:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829375.496681:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085816576:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787829375.496691:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829375.496697:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829375/real 1787829375] req@ffff9e35c498e940 x1874672085816576/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829400 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829375.496704:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e940 x1874672085816576/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829400 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829375.496710:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829375.496712:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829375.496714:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085816576:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829375.496717:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085816704:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829375.496727:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829375.496733:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829375/real 1787829375] req@ffff9e35c498f480 x1874672085816704/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829400 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829375.496740:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f480 x1874672085816704/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829400 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829375.496746:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829375.496748:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829375.496750:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085816704:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829375.496754:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085816832:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829375.496763:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829375.496769:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829375/real 1787829375] req@ffff9e35c498e1c0 x1874672085816832/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829400 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829375.496776:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e1c0 x1874672085816832/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829400 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829375.496782:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829375.496784:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829375.496786:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085816832:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829375.623922:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829375.623922:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829375.623922:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829375.624036:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829376.647897:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829376.647897:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829376.647917:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829376.647957:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829377.671899:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829377.671908:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829377.671952:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829377.671955:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829378.695895:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829378.695896:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829378.695898:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829378.695937:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829379.719870:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829379.719882:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787829379.719892:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829379.719971:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829380.615937:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829380.615948:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787829380.615952:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829380.615963:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2815 00000100:00080000:0.0:1787829380.615975:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829380.615982:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787829380.616024:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7192c0 x1874672085816960/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829380.616052:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829380.616061:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829380.616063:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829380.616069:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2815 00000100:00080000:0.0:1787829380.616073:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829380.616078:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829380.616091:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829380.616102:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085816960:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1787829380.616111:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718780 x1874672085817088/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829380.616126:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829380.616129:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787829380.616131:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829380.616137:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2815 00000100:00080000:0.0:1787829380.616142:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829380.616147:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829380.616152:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829380.616166:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a1c0 x1874672085817216/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787829380.616171:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787829380.616184:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829380.616188:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829380.616191:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829380.616197:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2815 00000100:00080000:0.0:1787829380.616202:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1787829380.616216:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829380/real 1787829380] req@ffff9e35da7192c0 x1874672085816960/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829405 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829380.616222:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829380.616232:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:1.0:1787829380.616237:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7192c0 x1874672085816960/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829405 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829380.616247:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1787829380.616249:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7183c0 x1874672085817344/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829380.616252:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829380.616260:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085816960:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1787829380.616270:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829380.616275:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829380.616277:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829380.616283:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2815 00000100:00100000:1.0:1787829380.616295:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085817088:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787829380.616296:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829380.616305:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829380.616312:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000800:00000100:1.0:1787829380.616317:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829380.616331:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a940 x1874672085817472/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829380.616335:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829380/real 1787829380] req@ffff9e35da718780 x1874672085817088/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829405 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829380.616364:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718780 x1874672085817088/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829405 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829380.616366:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829380.616377:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829380.616379:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829380.616381:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829380.616384:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829380.616390:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085817088:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787829380.616393:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2815 00000100:00100000:1.0:1787829380.616398:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085817216:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787829380.616401:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829380.616410:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:1.0:1787829380.616416:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:0.0:1787829380.616417:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829380.616424:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829380/real 1787829380] req@ffff9e35da71a1c0 x1874672085817216/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829405 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829380.616434:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718f00 x1874672085817600/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829380.616435:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a1c0 x1874672085817216/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829405 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829380.616466:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829380.616471:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:0.0:1787829380.616473:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787829380.616474:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085817216:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787829380.616482:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:1.0:1787829380.616483:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085817344:192.168.203.141@tcp:8:kworker.0 00000100:00080000:0.0:1787829380.616488:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829380.616496:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2815 00000800:00000100:1.0:1787829380.616501:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787829380.616502:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829380.616511:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787829380.616516:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829380/real 1787829380] req@ffff9e35da7183c0 x1874672085817344/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829405 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1787829380.616519:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829380.616538:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672085817344/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829405 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829380.616541:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718000 x1874672085817728/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829380.616563:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829380.616567:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829380.616571:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:0.0:1787829380.616588:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:1.0:1787829380.616592:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085817344:192.168.203.141@tcp:8:kworker.0 00000100:00080000:0.0:1787829380.616594:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829380.616606:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2815 00000100:00080000:0.0:1787829380.616611:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829380.616616:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787829380.616619:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085817472:192.168.203.141@tcp:8:kworker.0 00000008:00100000:0.0:1787829380.616625:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000800:00000100:1.0:1787829380.616644:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829380.616645:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a580 x1874672085817856/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829380.616655:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829380/real 1787829380] req@ffff9e35da71a940 x1874672085817472/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829405 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829380.616660:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829380.616666:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:1.0:1787829380.616668:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672085817472/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829405 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829380.616670:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829380.616683:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2815 00000100:00080000:1.0:1787829380.616684:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829380.616690:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829380.616691:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829380.616697:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085817472:192.168.203.141@tcp:8:kworker.0 00000100:00080000:0.0:1787829380.616700:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787829380.616706:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085817600:192.168.203.141@tcp:38:kworker.0 00000008:00100000:0.0:1787829380.616710:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000800:00000100:1.0:1787829380.616721:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829380.616730:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da719a40 x1874672085817984/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829380.616732:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829380/real 1787829380] req@ffff9e35da718f00 x1874672085817600/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829405 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829380.616762:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672085817600/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829405 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829380.616785:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829380.616789:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829380.616792:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085817600:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829380.616833:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085817728:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787829380.616849:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829380.616856:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829380/real 1787829380] req@ffff9e35da718000 x1874672085817728/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829405 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829380.616864:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718000 x1874672085817728/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829405 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829380.616870:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829380.616872:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829380.616874:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085817728:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829380.616883:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085817856:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829380.616893:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829380.616899:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829380/real 1787829380] req@ffff9e35da71a580 x1874672085817856/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829405 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829380.616906:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672085817856/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829405 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829380.616912:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829380.616913:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829380.616916:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085817856:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829380.616920:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085817984:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829380.616929:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829380.616938:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829380/real 1787829380] req@ffff9e35da719a40 x1874672085817984/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829405 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829380.616951:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672085817984/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829405 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829380.616960:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829380.616963:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829380.616967:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085817984:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829380.743946:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829380.743952:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787829380.743952:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829380.744039:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829381.767931:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787829381.767949:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829381.767961:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829381.768029:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829382.791904:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829382.791904:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787829382.791904:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829382.791983:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829383.815901:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829383.815902:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829383.815903:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829383.815972:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829384.839903:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829384.840039:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829384.840111:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829384.840153:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00080000:0.0:1787829385.735897:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829385.735906:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787829385.735910:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829385.735918:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2821 00000100:00080000:0.0:1787829385.735928:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829385.735933:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787829385.735966:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718b40 x1874672085818112/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829385.735992:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829385.735996:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829385.735998:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829385.736002:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2821 00000100:00080000:0.0:1787829385.736006:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829385.736010:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829385.736027:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829385.736044:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da719a40 x1874672085818240/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829385.736053:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085818112:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1787829385.736056:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829385.736062:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787829385.736065:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829385.736069:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2821 00000100:00080000:0.0:1787829385.736071:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829385.736075:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829385.736079:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829385.736088:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a580 x1874672085818368/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829385.736095:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829385.736098:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829385.736099:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829385.736102:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2821 00000100:00080000:0.0:1787829385.736118:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829385.736121:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829385.736124:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:0.0:1787829385.736132:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718000 x1874672085818496/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829385.736142:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829385.736145:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829385.736147:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829385.736152:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2821 00000100:00080000:0.0:1787829385.736155:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829385.736160:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829385.736165:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:1.0:1787829385.736176:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085818240:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829385.736180:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718f00 x1874672085818624/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829385.736194:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829385.736198:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829385.736200:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829385.736203:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2821 00000100:00080000:0.0:1787829385.736207:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829385.736211:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829385.736216:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829385.736227:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a940 x1874672085818752/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829385.736236:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787829385.736239:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085818368:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787829385.736240:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787829385.736244:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829385.736252:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2821 00000100:00080000:0.0:1787829385.736256:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829385.736260:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829385.736265:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829385.736279:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085818496:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829385.736286:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7183c0 x1874672085818880/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829385.736297:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829385.736300:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829385.736303:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829385.736308:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2821 00000100:00080000:0.0:1787829385.736312:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1787829385.736318:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085818624:192.168.203.141@tcp:8:kworker.0 00000100:00080000:0.0:1787829385.736319:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829385.736329:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:0.0:1787829385.736345:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a1c0 x1874672085819008/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829385.736356:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085818752:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787829385.736358:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829385.736366:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829385.736368:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829385.736373:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2821 00000100:00080000:0.0:1787829385.736377:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829385.736381:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829385.736386:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:1.0:1787829385.736392:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085818880:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829385.736398:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718780 x1874672085819136/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829385.736425:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085819008:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829385.736449:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085819136:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829385.863897:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829385.863898:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829385.863908:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787829385.863947:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829386.887952:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829386.887964:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829386.887979:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:1.0:1787829386.888036:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829387.911924:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829387.911933:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829387.911959:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787829387.911980:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1787829388.808085:0:2398:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.141:988 00000400:00000100:3.0:1787829388.808132:0:2398:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.141@tcp at host 192.168.203.141 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1787829388.808151:0:2398:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1787829388.808203:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000800:00000100:3.0:1787829388.808220:0:2398:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1787829388.808232:0:2398:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1787829388.808248:0:2398:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1787829388.808262:0:2398:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1787829388.808274:0:2398:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:1.0:1787829388.808277:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829385/real 1787829388] req@ffff9e35da718780 x1874672085819136/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829410 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:3.0:1787829388.808305:0:2398:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:1.0:1787829388.808330:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718780 x1874672085819136/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829410 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:3.0:1787829388.808330:0:2398:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00080000:1.0:1787829388.808367:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829388.808374:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829388.808380:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085819136:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829388.808389:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829385/real 1787829388] req@ffff9e35da718f00 x1874672085818624/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829410 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829388.808404:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672085818624/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829410 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829388.808413:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829388.808416:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829388.808420:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085818624:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829388.808429:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829385/real 1787829388] req@ffff9e35da71a940 x1874672085818752/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829410 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829388.808442:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672085818752/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829410 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829388.808451:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829388.808455:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829388.808458:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085818752:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829388.808466:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829385/real 1787829388] req@ffff9e35da7183c0 x1874672085818880/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829410 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829388.808478:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672085818880/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829410 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829388.808486:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829388.808490:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829388.808494:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085818880:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829388.808502:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829385/real 1787829388] req@ffff9e35da719a40 x1874672085818240/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829410 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829388.808515:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672085818240/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829410 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829388.808540:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829388.808545:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829388.808549:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085818240:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829388.808558:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829385/real 1787829388] req@ffff9e35da71a580 x1874672085818368/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829410 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829388.808572:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672085818368/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829410 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829388.808580:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829388.808583:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829388.808586:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085818368:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829388.808593:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829385/real 1787829388] req@ffff9e35da718000 x1874672085818496/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829410 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829388.808604:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718000 x1874672085818496/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829410 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829388.808612:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829388.808616:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829388.808620:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085818496:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829388.808633:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829385/real 1787829388] req@ffff9e35da718b40 x1874672085818112/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829410 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829388.808646:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718b40 x1874672085818112/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829410 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829388.808654:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829388.808657:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829388.808660:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085818112:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1787829388.808709:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829385/real 1787829388] req@ffff9e35da71a1c0 x1874672085819008/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829410 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829388.808722:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a1c0 x1874672085819008/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829410 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829388.808731:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829388.808735:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829388.808738:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085819008:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829388.935919:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829388.935926:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829388.935938:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787829388.935972:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829389.959935:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829389.959936:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829389.960002:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787829389.960012:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829390.855936:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829390.855948:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829390.855953:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829390.855965:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2826 00000100:00080000:1.0:1787829390.855976:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829390.855985:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787829390.856033:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f43c0 x1874672085819264/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829390.856072:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829390.856078:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829390.856081:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829390.856088:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2826 00000100:00080000:1.0:1787829390.856094:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829390.856101:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829390.856116:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829390.856130:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085819264:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1787829390.856142:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f70c0 x1874672085819392/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829390.856164:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829390.856168:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829390.856171:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829390.856177:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2826 00000100:00080000:1.0:1787829390.856181:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829390.856188:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829390.856194:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829390.856206:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4b40 x1874672085819520/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829390.856215:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829390.856221:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000100:0.0:1787829390.856223:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787829390.856224:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829390.856231:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2826 00000100:00080000:1.0:1787829390.856237:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829390.856242:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829390.856247:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:1.0:1787829390.856261:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5e00 x1874672085819648/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829390.856271:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829390.856276:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829390.856279:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829390.856285:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2826 00000100:00080000:1.0:1787829390.856291:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829390.856297:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829390.856302:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:0.0:1787829390.856308:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829390/real 1787829390] req@ffff9e35da6f43c0 x1874672085819264/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829415 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829390.856318:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5a40 x1874672085819776/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829390.856333:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829390.856339:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:0.0:1787829390.856340:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f43c0 x1874672085819264/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829415 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829390.856344:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829390.856354:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2826 00000100:00080000:1.0:1787829390.856360:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829390.856369:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:0.0:1787829390.856378:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:1.0:1787829390.856383:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1787829390.856387:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829390.856396:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085819264:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1787829390.856402:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7840 x1874672085819904/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829390.856415:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829390.856419:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829390.856421:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829390.856425:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2826 00000100:00080000:1.0:1787829390.856430:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787829390.856436:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085819392:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787829390.856437:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829390.856448:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829390.856465:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f43c0 x1874672085820032/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829390.856477:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787829390.856480:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829390.856486:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829390.856488:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829390.856495:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2826 00000100:00100000:0.0:1787829390.856500:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829390/real 1787829390] req@ffff9e35da6f70c0 x1874672085819392/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829415 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829390.856517:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829390.856528:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829390.856538:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:0.0:1787829390.856544:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f70c0 x1874672085819392/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829415 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829390.856554:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f61c0 x1874672085820160/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829390.856559:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829390.856570:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:1.0:1787829390.856570:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829390.856578:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:0.0:1787829390.856580:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085819392:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787829390.856583:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787829390.856593:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085819520:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787829390.856593:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2826 00000100:00080000:1.0:1787829390.856599:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829390.856605:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829390.856610:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:1.0:1787829390.856626:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4f00 x1874672085820288/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829390.856636:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829390.856658:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829390/real 1787829390] req@ffff9e35da6f4b40 x1874672085819520/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829415 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829390.856673:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4b40 x1874672085819520/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829415 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829390.856682:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829390.856686:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829390.856690:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085819520:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829390.856698:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085819648:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829390.856721:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829390.856733:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829390/real 1787829390] req@ffff9e35da6f5e00 x1874672085819648/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829415 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829390.856745:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5e00 x1874672085819648/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829415 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829390.856754:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829390.856757:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829390.856760:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085819648:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829390.856767:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085819776:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829390.856789:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829390.856853:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829390/real 1787829390] req@ffff9e35da6f5a40 x1874672085819776/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829415 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829390.856868:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5a40 x1874672085819776/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829415 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829390.856878:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829390.856883:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829390.856895:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085819776:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829390.856904:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085819904:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829390.856937:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829390.856955:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829390/real 1787829390] req@ffff9e35da6f7840 x1874672085819904/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829415 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829390.856976:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7840 x1874672085819904/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829415 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829390.856986:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829390.856989:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829390.856992:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085819904:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829390.857020:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085820032:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829390.857042:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829390.857055:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829390/real 1787829390] req@ffff9e35da6f43c0 x1874672085820032/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829415 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829390.857069:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f43c0 x1874672085820032/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829415 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829390.857079:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829390.857082:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829390.857086:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085820032:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829390.857093:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085820160:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829390.857114:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829390.857127:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829390/real 1787829390] req@ffff9e35da6f61c0 x1874672085820160/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829415 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829390.857139:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f61c0 x1874672085820160/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829415 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829390.857155:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829390.857159:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829390.857163:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085820160:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829390.857170:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085820288:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829390.857198:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829390.857211:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829390/real 1787829390] req@ffff9e35da6f4f00 x1874672085820288/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829415 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829390.857224:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4f00 x1874672085820288/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829415 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829390.857238:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829390.857243:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829390.857247:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085820288:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829390.983908:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829390.983909:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829390.983938:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787829390.983991:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829392.007916:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829392.007963:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787829392.007986:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829392.008008:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829393.031927:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829393.031927:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829393.031927:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787829393.032000:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829394.055906:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829394.055906:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829394.055920:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:1.0:1787829394.055958:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829395.079891:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829395.079900:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829395.079946:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787829395.079960:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829395.975898:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829395.975909:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787829395.975914:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829395.975924:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2831 00000100:00080000:0.0:1787829395.975936:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829395.975944:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787829395.975991:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7192c0 x1874672085820416/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829395.976015:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829395.976018:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829395.976019:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829395.976022:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2831 00000100:00080000:0.0:1787829395.976025:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829395.976028:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829395.976038:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829395.976049:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a1c0 x1874672085820544/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829395.976057:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829395.976060:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787829395.976061:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829395.976067:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2831 00000100:00100000:1.0:1787829395.976067:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085820416:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1787829395.976073:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829395.976078:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829395.976081:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829395.976090:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718b40 x1874672085820672/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829395.976097:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829395.976099:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829395.976101:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829395.976105:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2831 00000100:00080000:0.0:1787829395.976107:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829395.976110:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829395.976114:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:0.0:1787829395.976121:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718000 x1874672085820800/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829395.976127:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829395.976130:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829395.976131:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829395.976134:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2831 00000100:00080000:0.0:1787829395.976137:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829395.976140:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:1.0:1787829395.976145:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:0.0:1787829395.976146:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:0.0:1787829395.976157:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a580 x1874672085820928/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829395.976164:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829395.976167:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829395.976169:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829395.976171:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2831 00000100:00080000:0.0:1787829395.976174:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829395.976176:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829395.976179:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829395.976189:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829395/real 1787829395] req@ffff9e35da7192c0 x1874672085820416/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829420 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829395.976191:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da719a40 x1874672085821056/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829395.976225:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829395.976229:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787829395.976230:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787829395.976233:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7192c0 x1874672085820416/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829420 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829395.976236:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2831 00000100:00080000:0.0:1787829395.976241:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829395.976248:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829395.976254:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00080000:1.0:1787829395.976256:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829395.976263:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829395.976264:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7183c0 x1874672085821184/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829395.976270:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085820416:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1787829395.976277:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829395.976280:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829395.976282:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829395.976286:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2831 00000100:00080000:0.0:1787829395.976289:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829395.976291:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829395.976295:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:1.0:1787829395.976299:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085820544:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829395.976305:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a940 x1874672085821312/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787829395.976319:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787829395.976320:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829395.976325:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829395.976327:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829395.976332:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2831 00000100:00100000:1.0:1787829395.976334:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829395/real 1787829395] req@ffff9e35da71a1c0 x1874672085820544/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829420 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829395.976337:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829395.976344:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829395.976352:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:1.0:1787829395.976354:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a1c0 x1874672085820544/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829420 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829395.976363:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1787829395.976364:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718f00 x1874672085821440/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829395.976371:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829395.976378:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085820544:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829395.976391:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085820672:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787829395.976408:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829395.976415:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829395/real 1787829395] req@ffff9e35da718b40 x1874672085820672/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829420 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829395.976424:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718b40 x1874672085820672/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829420 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829395.976437:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829395.976439:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829395.976444:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085820672:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829395.976448:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085820800:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829395.976479:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829395.976492:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829395/real 1787829395] req@ffff9e35da718000 x1874672085820800/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829420 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829395.976506:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718000 x1874672085820800/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829420 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829395.976516:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829395.976519:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829395.976523:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085820800:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829395.976529:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085820928:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829395.976547:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829395.976555:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829395/real 1787829395] req@ffff9e35da71a580 x1874672085820928/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829420 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829395.976568:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672085820928/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829420 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829395.976575:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829395.976578:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829395.976581:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085820928:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829395.976586:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085821056:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787829395.976599:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829395.976606:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829395/real 1787829395] req@ffff9e35da719a40 x1874672085821056/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829420 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829395.976615:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672085821056/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829420 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829395.976621:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829395.976623:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829395.976624:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085821056:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829395.976631:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085821184:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787829395.976642:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829395.976648:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829395/real 1787829395] req@ffff9e35da7183c0 x1874672085821184/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829420 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829395.976655:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672085821184/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829420 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829395.976668:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829395.976671:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829395.976675:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085821184:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829395.976701:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085821312:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829395.976722:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829395.976731:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829395/real 1787829395] req@ffff9e35da71a940 x1874672085821312/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829420 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829395.976739:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672085821312/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829420 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829395.976745:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829395.976748:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829395.976750:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085821312:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829395.976754:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085821440:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829395.976765:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829395.976771:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829395/real 1787829395] req@ffff9e35da718f00 x1874672085821440/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829420 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829395.976779:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672085821440/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829420 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829395.976784:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829395.976786:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829395.976788:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085821440:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829396.103941:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829396.103941:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829396.103960:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829396.104009:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829397.127873:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829397.127893:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:3.0:1787829397.127903:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787829397.128018:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829398.151941:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:3.0:1787829398.151945:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829398.151946:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787829398.152074:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829399.175925:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829399.175963:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829399.175969:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829399.175987:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829400.199919:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829400.199940:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829400.199959:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787829400.199969:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829401.095944:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829401.095959:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787829401.095964:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829401.095977:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2836 00000100:00080000:0.0:1787829401.095991:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829401.095999:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787829401.096043:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718780 x1874672085821568/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829401.096075:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829401.096079:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829401.096082:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829401.096087:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2836 00000100:00080000:0.0:1787829401.096091:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829401.096096:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829401.096110:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829401.096127:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085821568:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1787829401.096128:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718f00 x1874672085821696/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829401.096143:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829401.096146:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787829401.096148:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829401.096153:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2836 00000100:00080000:0.0:1787829401.096158:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829401.096162:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829401.096168:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829401.096183:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a940 x1874672085821824/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829401.096194:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1787829401.096200:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787829401.096201:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829401.096204:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829401.096209:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2836 00000100:00080000:0.0:1787829401.096214:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829401.096218:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829401.096223:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:0.0:1787829401.096234:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7183c0 x1874672085821952/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829401.096241:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829401.096244:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829401.096246:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829401.096249:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2836 00000100:00080000:0.0:1787829401.096255:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1787829401.096255:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829401/real 1787829401] req@ffff9e35da718780 x1874672085821568/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829426 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829401.096261:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829401.096268:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:0.0:1787829401.096278:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da719a40 x1874672085822080/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829401.096289:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829401.096291:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829401.096293:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787829401.096296:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718780 x1874672085821568/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829426 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829401.096299:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2836 00000100:00080000:0.0:1787829401.096304:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829401.096310:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829401.096316:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00080000:1.0:1787829401.096317:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829401.096322:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829401.096325:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a580 x1874672085822208/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829401.096330:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085821568:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1787829401.096342:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829401.096345:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787829401.096347:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829401.096351:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2836 00000100:00080000:0.0:1787829401.096353:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829401.096355:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829401.096358:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829401.096362:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085821696:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829401.096370:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718000 x1874672085822336/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829401.096379:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829401.096382:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829401.096384:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1787829401.096388:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787829401.096389:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2836 00000100:00080000:0.0:1787829401.096395:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829401.096399:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829401.096403:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:1.0:1787829401.096405:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829401/real 1787829401] req@ffff9e35da718f00 x1874672085821696/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829426 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829401.096414:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718b40 x1874672085822464/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829401.096420:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672085821696/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829426 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829401.096427:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829401.096444:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829401.096447:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829401.096456:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2836 00000100:00080000:1.0:1787829401.096457:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829401.096462:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829401.096464:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:0.0:1787829401.096469:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787829401.096472:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085821696:192.168.203.141@tcp:38:kworker.0 00000008:00100000:0.0:1787829401.096476:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:1.0:1787829401.096481:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085821824:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829401.096490:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a1c0 x1874672085822592/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787829401.096503:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829401.096520:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829401/real 1787829401] req@ffff9e35da71a940 x1874672085821824/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829426 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829401.096532:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672085821824/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829426 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829401.096538:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829401.096540:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829401.096543:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085821824:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829401.096547:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085821952:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829401.096560:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829401.096566:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829401/real 1787829401] req@ffff9e35da7183c0 x1874672085821952/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829426 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829401.096574:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672085821952/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829426 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829401.096580:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829401.096582:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829401.096583:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085821952:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829401.096587:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085822080:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829401.096597:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829401.096603:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829401/real 1787829401] req@ffff9e35da719a40 x1874672085822080/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829426 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829401.096611:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672085822080/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829426 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829401.096616:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829401.096618:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829401.096620:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085822080:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829401.096623:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085822208:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787829401.096634:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829401.096640:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829401/real 1787829401] req@ffff9e35da71a580 x1874672085822208/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829426 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829401.096647:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672085822208/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829426 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829401.096652:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829401.096654:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829401.096656:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085822208:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829401.096670:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085822336:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787829401.096680:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829401.096686:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829401/real 1787829401] req@ffff9e35da718000 x1874672085822336/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829426 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829401.096694:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718000 x1874672085822336/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829426 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829401.096700:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829401.096702:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829401.096704:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085822336:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829401.096707:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085822464:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829401.096716:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829401.096722:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829401/real 1787829401] req@ffff9e35da718b40 x1874672085822464/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829426 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829401.096729:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718b40 x1874672085822464/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829426 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829401.096734:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829401.096736:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829401.096739:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085822464:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829401.096744:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085822592:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829401.096761:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829401.096773:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829401/real 1787829401] req@ffff9e35da71a1c0 x1874672085822592/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829426 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829401.096786:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a1c0 x1874672085822592/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829426 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829401.096796:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829401.096821:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829401.096824:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085822592:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829401.223934:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829401.223935:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829401.223944:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:0.0:1787829401.224035:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829402.247937:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829402.247938:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829402.247957:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829402.247973:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829403.271926:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829403.271926:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829403.271938:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787829403.271981:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829404.295932:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829404.295932:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829404.295932:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787829404.296005:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829405.319907:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829405.319915:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829405.319935:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829405.319963:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829406.215961:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829406.215974:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829406.215979:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829406.215993:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2841 00000100:00080000:1.0:1787829406.216005:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829406.216014:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787829406.216064:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6580 x1874672085822720/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829406.216093:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829406.216096:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829406.216097:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829406.216100:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2841 00000100:00080000:1.0:1787829406.216104:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829406.216108:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829406.216119:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829406.216135:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085822720:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1787829406.216137:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4780 x1874672085822848/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829406.216162:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829406.216166:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829406.216168:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829406.216171:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2841 00000100:00080000:1.0:1787829406.216195:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829406.216198:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829406.216203:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1787829406.216205:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829406.216218:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6d00 x1874672085822976/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829406.216226:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829406.216229:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829406.216231:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829406.216234:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2841 00000100:00080000:1.0:1787829406.216237:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829406.216239:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829406.216243:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:1.0:1787829406.216253:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6940 x1874672085823104/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829406.216261:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787829406.216262:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829406/real 1787829406] req@ffff9e35da6f6580 x1874672085822720/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829431 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829406.216266:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829406.216269:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829406.216276:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2841 00000100:00080000:1.0:1787829406.216282:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829406.216285:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829406.216288:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:0.0:1787829406.216296:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6580 x1874672085822720/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829431 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829406.216299:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4000 x1874672085823232/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829406.216325:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829406.216327:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829406.216328:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829406.216332:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829406.216335:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:1.0:1787829406.216338:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2841 00000100:00080000:1.0:1787829406.216342:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787829406.216343:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085822720:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787829406.216347:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829406.216352:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829406.216361:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7480 x1874672085823360/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829406.216366:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829406.216369:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829406.216371:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787829406.216376:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085822848:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787829406.216376:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2841 00000100:00080000:1.0:1787829406.216381:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829406.216385:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829406.216388:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829406.216417:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6580 x1874672085823488/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829406.216425:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787829406.216436:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787829406.216441:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829406/real 1787829406] req@ffff9e35da6f4780 x1874672085822848/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829431 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829406.216442:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829406.216446:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829406.216453:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2841 00000100:00080000:1.0:1787829406.216460:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787829406.216464:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4780 x1874672085822848/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829431 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829406.216465:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829406.216472:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00080000:0.0:1787829406.216480:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829406.216485:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829406.216487:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4f00 x1874672085823616/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829406.216491:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085822848:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787829406.216503:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787829406.216504:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085822976:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787829406.216509:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829406.216512:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829406.216515:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2841 00000100:00080000:1.0:1787829406.216518:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829406.216522:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:0.0:1787829406.216528:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:1.0:1787829406.216528:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:0.0:1787829406.216539:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829406/real 1787829406] req@ffff9e35da6f6d00 x1874672085822976/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829431 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829406.216548:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f61c0 x1874672085823744/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829406.216553:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6d00 x1874672085822976/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829431 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829406.216574:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829406.216577:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829406.216581:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085822976:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829406.216585:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085823104:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829406.216599:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829406.216608:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829406/real 1787829406] req@ffff9e35da6f6940 x1874672085823104/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829431 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829406.216616:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6940 x1874672085823104/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829431 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829406.216621:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829406.216623:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829406.216625:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085823104:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829406.216629:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085823232:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829406.216639:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829406.216645:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829406/real 1787829406] req@ffff9e35da6f4000 x1874672085823232/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829431 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829406.216652:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4000 x1874672085823232/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829431 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829406.216658:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829406.216660:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829406.216662:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085823232:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829406.216666:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085823360:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829406.216676:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829406.216682:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829406/real 1787829406] req@ffff9e35da6f7480 x1874672085823360/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829431 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829406.216689:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7480 x1874672085823360/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829431 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829406.216694:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829406.216697:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829406.216698:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085823360:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829406.216717:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085823488:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829406.216737:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829406.216749:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829406/real 1787829406] req@ffff9e35da6f6580 x1874672085823488/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829431 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829406.216763:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6580 x1874672085823488/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829431 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829406.216771:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829406.216773:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829406.216776:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085823488:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829406.216781:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085823616:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829406.216795:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829406.216822:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829406/real 1787829406] req@ffff9e35da6f4f00 x1874672085823616/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829431 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829406.216830:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4f00 x1874672085823616/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829431 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829406.216836:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829406.216838:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829406.216840:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085823616:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829406.216844:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085823744:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829406.216856:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829406.216862:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829406/real 1787829406] req@ffff9e35da6f61c0 x1874672085823744/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829431 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829406.216870:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f61c0 x1874672085823744/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829431 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829406.216875:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829406.216877:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829406.216879:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085823744:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829406.343920:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829406.343920:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829406.343932:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787829406.343983:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829407.367920:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829407.367935:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829407.367949:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787829407.367970:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829408.391896:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829408.391897:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829408.391914:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829408.391973:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829409.415902:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829409.415902:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829409.415938:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787829409.415999:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829410.439901:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829410.439901:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829410.439942:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787829410.439975:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829411.335927:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829411.335939:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787829411.335943:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829411.335954:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2846 00000100:00080000:3.0:1787829411.335964:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829411.335972:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787829411.336020:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499680 x1874672085823872/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829411.336058:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829411.336062:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829411.336063:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829411.336068:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2846 00000100:00080000:3.0:1787829411.336077:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829411.336080:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829411.336093:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829411.336105:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b840 x1874672085824000/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829411.336115:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829411.336117:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787829411.336119:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829411.336123:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2846 00000100:00080000:3.0:1787829411.336126:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829411.336128:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829411.336131:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829411.336142:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649bc00 x1874672085824128/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829411.336144:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085823872:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1787829411.336158:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829411.336160:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787829411.336162:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829411.336164:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2846 00000100:00080000:3.0:1787829411.336166:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829411.336169:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829411.336172:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:3.0:1787829411.336181:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498b40 x1874672085824256/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829411.336187:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829411.336189:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787829411.336190:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829411.336193:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2846 00000100:00080000:3.0:1787829411.336196:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829411.336199:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829411.336202:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:3.0:1787829411.336212:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a940 x1874672085824384/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829411.336218:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1787829411.336224:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1787829411.336224:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829411.336228:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829411.336232:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2846 00000100:00080000:3.0:1787829411.336235:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829411.336238:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829411.336242:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829411.336251:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498780 x1874672085824512/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829411.336257:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829411.336260:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787829411.336262:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829411.336265:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2846 00000100:00080000:3.0:1787829411.336268:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787829411.336273:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829411/real 1787829411] req@ffff9e35c6499680 x1874672085823872/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829436 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829411.336273:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829411.336280:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829411.336293:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499a40 x1874672085824640/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829411.336298:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499680 x1874672085823872/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829436 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829411.336329:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829411.336330:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829411.336336:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829411.336337:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:3.0:1787829411.336339:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787829411.336343:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085823872:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1787829411.336345:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2846 00000100:00080000:3.0:1787829411.336351:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787829411.336355:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085824000:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1787829411.336356:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829411.336375:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:3.0:1787829411.336390:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498f00 x1874672085824768/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829411.336397:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1787829411.336399:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1787829411.336404:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787829411.336406:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829411.336411:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2846 00000100:00100000:0.0:1787829411.336415:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829411/real 1787829411] req@ffff9e35c649b840 x1874672085824000/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829436 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829411.336417:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829411.336424:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829411.336430:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:0.0:1787829411.336432:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b840 x1874672085824000/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829436 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829411.336441:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1787829411.336442:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a1c0 x1874672085824896/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829411.336447:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829411.336455:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085824000:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829411.336484:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085824128:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829411.336498:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829411.336505:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829411/real 1787829411] req@ffff9e35c649bc00 x1874672085824128/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829436 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829411.336513:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649bc00 x1874672085824128/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829436 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829411.336519:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829411.336521:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829411.336523:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085824128:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829411.336527:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085824256:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829411.336536:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829411.336541:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829411/real 1787829411] req@ffff9e35c6498b40 x1874672085824256/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829436 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829411.336549:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498b40 x1874672085824256/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829436 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829411.336554:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829411.336556:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829411.336558:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085824256:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829411.336561:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085824384:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829411.336571:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829411.336577:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829411/real 1787829411] req@ffff9e35c649a940 x1874672085824384/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829436 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829411.336589:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a940 x1874672085824384/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829436 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829411.336594:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829411.336596:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829411.336598:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085824384:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829411.336601:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085824512:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829411.336612:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829411.336617:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829411/real 1787829411] req@ffff9e35c6498780 x1874672085824512/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829436 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829411.336625:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498780 x1874672085824512/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829436 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829411.336630:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829411.336632:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829411.336634:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085824512:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829411.336637:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085824640:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829411.336646:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829411.336652:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829411/real 1787829411] req@ffff9e35c6499a40 x1874672085824640/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829436 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829411.336659:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499a40 x1874672085824640/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829436 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829411.336668:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829411.336670:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829411.336672:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085824640:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829411.336675:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085824768:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829411.336684:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829411.336690:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829411/real 1787829411] req@ffff9e35c6498f00 x1874672085824768/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829436 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829411.336696:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498f00 x1874672085824768/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829436 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829411.336702:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829411.336704:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829411.336706:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085824768:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829411.336709:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085824896:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829411.336723:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829411.336733:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829411/real 1787829411] req@ffff9e35c649a1c0 x1874672085824896/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829436 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829411.336744:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a1c0 x1874672085824896/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829436 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829411.336753:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829411.336756:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829411.336761:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085824896:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829411.463936:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829411.463936:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787829411.464058:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829411.464118:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829412.487881:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829412.487890:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829412.487910:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787829412.487989:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829413.511926:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829413.511926:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829413.511926:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:0.0:1787829413.511979:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829414.535917:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829414.535927:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829414.535958:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829414.535979:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829415.559931:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829415.559941:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829415.559962:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829415.559986:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829416.455933:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829416.455948:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787829416.455953:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829416.455967:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2851 00000100:00080000:3.0:1787829416.455981:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829416.455990:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787829416.456036:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c64983c0 x1874672085825024/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829416.456065:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829416.456068:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829416.456070:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829416.456073:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2851 00000100:00080000:3.0:1787829416.456076:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829416.456080:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829416.456089:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829416.456098:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649ad00 x1874672085825152/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829416.456106:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829416.456109:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787829416.456110:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829416.456113:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2851 00000100:00080000:3.0:1787829416.456115:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829416.456117:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829416.456120:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829416.456127:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498000 x1874672085825280/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829416.456133:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c64983c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085825024:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1787829416.456137:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829416.456144:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787829416.456146:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829416.456149:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2851 00000100:00080000:3.0:1787829416.456151:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829416.456154:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829416.456156:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:3.0:1787829416.456166:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499e00 x1874672085825408/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829416.456172:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829416.456175:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787829416.456176:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829416.456179:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2851 00000100:00080000:3.0:1787829416.456181:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829416.456183:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829416.456186:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:3.0:1787829416.456194:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c64992c0 x1874672085825536/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829416.456200:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829416.456202:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829416.456203:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829416.456206:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2851 00000100:00080000:3.0:1787829416.456209:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829416.456212:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829416.456215:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829416.456225:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b480 x1874672085825664/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829416.456227:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1787829416.456236:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829416.456239:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787829416.456241:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829416.456243:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2851 00000100:00080000:3.0:1787829416.456245:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829416.456248:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829416.456251:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829416.456258:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b0c0 x1874672085825792/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829416.456263:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829416.456266:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787829416.456267:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829416.456270:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2851 00000100:00080000:3.0:1787829416.456273:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829416.456276:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829416.456279:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:3.0:1787829416.456288:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a580 x1874672085825920/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829416.456293:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829416/real 1787829416] req@ffff9e35c64983c0 x1874672085825024/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829441 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829416.456318:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829416.456322:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787829416.456338:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829416.456342:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2851 00000100:00080000:3.0:1787829416.456345:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829416.456351:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787829416.456353:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c64983c0 x1874672085825024/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829441 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1787829416.456355:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:3.0:1787829416.456371:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a1c0 x1874672085826048/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829416.456372:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829416.456384:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829416.456392:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c64983c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085825024:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1787829416.456403:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085825152:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829416.456432:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829416.456446:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829416/real 1787829416] req@ffff9e35c649ad00 x1874672085825152/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829441 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829416.456460:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649ad00 x1874672085825152/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829441 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829416.456470:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829416.456474:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829416.456478:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085825152:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829416.456507:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085825280:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829416.456530:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829416.456543:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829416/real 1787829416] req@ffff9e35c6498000 x1874672085825280/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829441 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829416.456555:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498000 x1874672085825280/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829441 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829416.456563:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829416.456566:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829416.456569:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085825280:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829416.456576:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085825408:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829416.456596:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829416.456606:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829416/real 1787829416] req@ffff9e35c6499e00 x1874672085825408/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829441 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829416.456618:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499e00 x1874672085825408/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829441 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829416.456627:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829416.456631:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829416.456634:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085825408:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829416.456641:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c64992c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085825536:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829416.456662:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829416.456675:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829416/real 1787829416] req@ffff9e35c64992c0 x1874672085825536/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829441 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829416.456688:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c64992c0 x1874672085825536/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829441 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829416.456698:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829416.456701:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829416.456705:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c64992c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085825536:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829416.456712:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085825664:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829416.456732:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829416.456745:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829416/real 1787829416] req@ffff9e35c649b480 x1874672085825664/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829441 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829416.456770:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b480 x1874672085825664/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829441 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829416.456780:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829416.456784:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829416.456787:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085825664:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829416.456794:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085825792:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829416.456864:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829416.456877:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829416/real 1787829416] req@ffff9e35c649b0c0 x1874672085825792/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829441 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829416.456889:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b0c0 x1874672085825792/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829441 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829416.456899:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829416.456912:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829416.456916:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085825792:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829416.456923:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085825920:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829416.456941:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829416.456951:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829416/real 1787829416] req@ffff9e35c649a580 x1874672085825920/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829441 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829416.456964:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a580 x1874672085825920/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829441 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829416.456975:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829416.456978:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829416.456982:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085825920:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829416.456988:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085826048:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829416.457016:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829416.457029:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829416/real 1787829416] req@ffff9e35c649a1c0 x1874672085826048/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829441 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829416.457041:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a1c0 x1874672085826048/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829441 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829416.457050:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829416.457054:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829416.457058:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085826048:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829416.583876:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829416.583878:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829416.583891:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:0.0:1787829416.583986:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829417.607958:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829417.607963:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829417.607966:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829417.608045:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829418.631880:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829418.631881:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829418.631882:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829418.631941:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829419.655891:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829419.655891:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829419.655909:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:1.0:1787829419.655956:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829420.679901:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829420.679901:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829420.679957:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829420.680048:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00080000:0.0:1787829421.575914:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829421.575928:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787829421.575931:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829421.575940:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2856 00000100:00080000:0.0:1787829421.575949:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829421.575955:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787829421.575990:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7192c0 x1874672085826176/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829421.576021:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829421.576026:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829421.576028:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829421.576033:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2856 00000100:00080000:0.0:1787829421.576037:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829421.576042:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829421.576053:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829421.576069:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085826176:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1787829421.576072:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a1c0 x1874672085826304/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829421.576088:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829421.576092:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787829421.576095:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829421.576099:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2856 00000100:00080000:0.0:1787829421.576104:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829421.576109:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829421.576114:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829421.576129:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718b40 x1874672085826432/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787829421.576141:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787829421.576142:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829421.576147:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829421.576150:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829421.576157:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2856 00000100:00080000:0.0:1787829421.576162:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829421.576167:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829421.576173:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:1.0:1787829421.576181:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829421/real 1787829421] req@ffff9e35da7192c0 x1874672085826176/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829446 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829421.576189:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718000 x1874672085826560/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829421.576200:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7192c0 x1874672085826176/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829446 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829421.576203:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829421.576210:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829421.576213:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829421.576217:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829421.576220:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2856 00000100:00080000:1.0:1787829421.576224:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:0.0:1787829421.576226:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1787829421.576230:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085826176:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1787829421.576233:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829421.576240:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:0.0:1787829421.576250:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a580 x1874672085826688/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829421.576258:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787829421.576258:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085826304:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787829421.576265:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829421.576267:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829421.576271:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2856 00000100:00080000:0.0:1787829421.576274:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829421.576279:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829421.576285:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1787829421.576285:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829421.576316:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da719a40 x1874672085826816/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829421.576318:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829421/real 1787829421] req@ffff9e35da71a1c0 x1874672085826304/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829446 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829421.576345:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787829421.576347:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a1c0 x1874672085826304/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829446 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829421.576351:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787829421.576354:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829421.576362:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2856 00000100:00080000:0.0:1787829421.576368:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829421.576372:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829421.576374:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:1.0:1787829421.576379:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000008:00100000:0.0:1787829421.576382:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829421.576383:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085826304:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829421.576393:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085826432:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829421.576397:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7183c0 x1874672085826944/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829421.576406:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829421.576409:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829421.576410:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1787829421.576413:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787829421.576417:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2856 00000100:00080000:0.0:1787829421.576421:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1787829421.576425:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829421/real 1787829421] req@ffff9e35da718b40 x1874672085826432/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829446 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829421.576426:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829421.576434:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:1.0:1787829421.576445:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718b40 x1874672085826432/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829446 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829421.576450:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a940 x1874672085827072/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829421.576455:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829421.576462:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829421.576468:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085826432:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787829421.576473:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787829421.576476:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085826560:192.168.203.141@tcp:8:kworker.0 00000100:00080000:0.0:1787829421.576480:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829421.576484:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829421.576490:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2856 00000100:00080000:0.0:1787829421.576494:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1787829421.576495:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787829421.576503:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787829421.576507:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829421/real 1787829421] req@ffff9e35da718000 x1874672085826560/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829446 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1787829421.576512:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:1.0:1787829421.576519:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718000 x1874672085826560/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829446 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829421.576526:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829421.576530:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829421.576532:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718f00 x1874672085827200/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829421.576534:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085826560:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829421.576557:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085826688:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829421.576580:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829421.576593:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829421/real 1787829421] req@ffff9e35da71a580 x1874672085826688/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829446 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829421.576606:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672085826688/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829446 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829421.576614:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829421.576616:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829421.576618:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085826688:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829421.576622:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085826816:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787829421.576637:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829421.576649:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829421/real 1787829421] req@ffff9e35da719a40 x1874672085826816/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829446 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829421.576661:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672085826816/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829446 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829421.576670:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829421.576674:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829421.576677:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085826816:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829421.576684:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085826944:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787829421.576703:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829421.576716:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829421/real 1787829421] req@ffff9e35da7183c0 x1874672085826944/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829446 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829421.576728:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672085826944/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829446 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829421.576738:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829421.576741:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829421.576745:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085826944:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829421.576751:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085827072:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829421.576771:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829421.576784:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829421/real 1787829421] req@ffff9e35da71a940 x1874672085827072/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829446 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829421.576792:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672085827072/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829446 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829421.576821:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829421.576824:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829421.576826:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085827072:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829421.576830:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085827200:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829421.576849:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829421.576861:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829421/real 1787829421] req@ffff9e35da718f00 x1874672085827200/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829446 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829421.576874:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672085827200/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829446 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829421.576884:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829421.576887:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829421.576890:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085827200:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829421.703891:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829421.703891:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829421.703921:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829421.703943:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829422.727939:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829422.727951:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829422.727999:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787829422.728017:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829423.751893:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829423.751921:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787829423.752038:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829423.752117:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829424.775952:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787829424.775952:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829424.775958:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829424.776057:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829425.799934:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829425.799934:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:3.0:1787829425.799934:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:1.0:1787829425.800034:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829426.695922:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829426.695938:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787829426.695943:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829426.695954:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2862 00000100:00080000:0.0:1787829426.695965:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829426.695971:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787829426.696025:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718780 x1874672085827328/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829426.696056:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829426.696061:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829426.696063:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829426.696067:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2862 00000100:00080000:0.0:1787829426.696072:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829426.696077:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829426.696089:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829426.696100:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085827328:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1787829426.696109:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718f00 x1874672085827456/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829426.696122:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829426.696126:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787829426.696128:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829426.696133:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2862 00000100:00080000:0.0:1787829426.696137:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829426.696141:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829426.696148:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829426.696161:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a940 x1874672085827584/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787829426.696171:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787829426.696174:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829426.696179:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829426.696181:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829426.696188:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2862 00000100:00080000:0.0:1787829426.696193:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829426.696198:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829426.696205:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:0.0:1787829426.696219:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7183c0 x1874672085827712/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829426.696229:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829426/real 1787829426] req@ffff9e35da718780 x1874672085827328/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829451 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829426.696232:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829426.696243:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829426.696246:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787829426.696268:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718780 x1874672085827328/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829451 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829426.696271:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2862 00000100:00080000:0.0:1787829426.696278:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829426.696288:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:1.0:1787829426.696290:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:0.0:1787829426.696295:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00080000:1.0:1787829426.696298:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829426.696305:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085827328:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1787829426.696313:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da719a40 x1874672085827840/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829426.696325:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829426.696329:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829426.696331:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787829426.696337:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085827456:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787829426.696338:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2862 00000100:00080000:0.0:1787829426.696347:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829426.696353:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829426.696360:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829426.696374:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a580 x1874672085827968/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787829426.696376:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787829426.696390:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829426.696396:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:1.0:1787829426.696397:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829426/real 1787829426] req@ffff9e35da718f00 x1874672085827456/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829451 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829426.696401:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829426.696411:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2862 00000100:00080000:0.0:1787829426.696418:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829426.696426:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829426.696434:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829426.696441:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672085827456/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829451 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829426.696451:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718000 x1874672085828096/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829426.696456:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829426.696465:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:0.0:1787829426.696468:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787829426.696473:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085827456:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787829426.696475:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829426.696479:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829426.696486:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2862 00000100:00100000:1.0:1787829426.696488:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085827584:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787829426.696491:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829426.696499:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829426.696505:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:0.0:1787829426.696516:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718b40 x1874672085828224/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829426.696527:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829426.696531:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000800:00000100:1.0:1787829426.696535:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787829426.696536:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829426.696549:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2862 00000100:00100000:1.0:1787829426.696554:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829426/real 1787829426] req@ffff9e35da71a940 x1874672085827584/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829451 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829426.696556:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829426.696565:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829426.696570:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:0.0:1787829426.696591:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a1c0 x1874672085828352/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829426.696592:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672085827584/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829451 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829426.696626:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829426.696632:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829426.696637:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085827584:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829426.696644:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085827712:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829426.696673:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829426.696695:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829426/real 1787829426] req@ffff9e35da7183c0 x1874672085827712/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829451 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829426.696716:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672085827712/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829451 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829426.696732:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829426.696737:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829426.696743:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085827712:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829426.696757:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085827840:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829426.696782:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829426.696796:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829426/real 1787829426] req@ffff9e35da719a40 x1874672085827840/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829451 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829426.696853:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672085827840/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829451 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829426.696864:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829426.696869:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829426.696874:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085827840:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829426.696901:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085827968:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787829426.696926:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829426.696952:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829426/real 1787829426] req@ffff9e35da71a580 x1874672085827968/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829451 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829426.696968:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672085827968/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829451 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829426.696977:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829426.696981:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829426.696986:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085827968:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829426.696996:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085828096:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787829426.697018:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829426.697033:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829426/real 1787829426] req@ffff9e35da718000 x1874672085828096/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829451 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829426.697046:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718000 x1874672085828096/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829451 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829426.697057:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829426.697061:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829426.697065:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085828096:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829426.697073:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085828224:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829426.697096:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829426.697112:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829426/real 1787829426] req@ffff9e35da718b40 x1874672085828224/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829451 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829426.697124:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718b40 x1874672085828224/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829451 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829426.697134:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829426.697137:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829426.697141:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085828224:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829426.697147:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085828352:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829426.697169:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829426.697182:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829426/real 1787829426] req@ffff9e35da71a1c0 x1874672085828352/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829451 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829426.697197:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a1c0 x1874672085828352/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829451 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829426.697206:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829426.697211:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829426.697216:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085828352:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829426.823925:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829426.823978:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829426.824092:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829426.824093:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829427.847945:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829427.847977:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829427.847988:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829427.848001:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829428.871944:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829428.871961:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829428.871961:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829428.871985:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829429.895971:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829429.895972:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829429.895972:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829429.896020:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829430.919920:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829430.919934:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829430.919961:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829430.919966:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787829431.815914:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829431.815926:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829431.815931:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829431.815941:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2867 00000100:00080000:1.0:1787829431.815953:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829431.815960:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787829431.816002:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f43c0 x1874672085828480/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829431.816032:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829431.816035:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829431.816037:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829431.816041:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2867 00000100:00080000:1.0:1787829431.816044:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829431.816047:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829431.816057:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829431.816068:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7840 x1874672085828608/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829431.816077:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787829431.816079:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085828480:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787829431.816081:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829431.816084:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829431.816090:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2867 00000100:00080000:1.0:1787829431.816093:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829431.816095:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829431.816099:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829431.816108:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5a40 x1874672085828736/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829431.816115:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829431.816118:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829431.816119:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829431.816122:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2867 00000100:00080000:1.0:1787829431.816141:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829431.816143:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829431.816146:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000800:00000100:0.0:1787829431.816152:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829431.816156:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5e00 x1874672085828864/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829431.816169:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829431.816172:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829431.816173:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829431.816176:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2867 00000100:00080000:1.0:1787829431.816178:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829431.816181:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829431.816184:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:1.0:1787829431.816192:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4b40 x1874672085828992/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829431.816199:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787829431.816201:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829431/real 1787829431] req@ffff9e35da6f43c0 x1874672085828480/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829456 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829431.816202:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829431.816206:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829431.816213:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2867 00000100:00080000:1.0:1787829431.816233:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829431.816237:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829431.816241:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829431.816244:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f43c0 x1874672085828480/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829456 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829431.816254:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f70c0 x1874672085829120/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829431.816257:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829431.816267:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:1.0:1787829431.816269:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829431.816274:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:0.0:1787829431.816275:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085828480:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787829431.816277:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829431.816286:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2867 00000100:00080000:1.0:1787829431.816291:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829431.816296:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829431.816301:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829431.816308:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085828608:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829431.816316:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f43c0 x1874672085829248/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829431.816330:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1787829431.816334:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787829431.816338:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829431.816342:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787829431.816349:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829431/real 1787829431] req@ffff9e35da6f7840 x1874672085828608/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829456 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829431.816350:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2867 00000100:00080000:1.0:1787829431.816359:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787829431.816367:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7840 x1874672085828608/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829456 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829431.816368:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829431.816379:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00080000:0.0:1787829431.816381:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829431.816389:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829431.816392:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085828608:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829431.816401:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f61c0 x1874672085829376/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829431.816402:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085828736:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787829431.816419:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1787829431.816424:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787829431.816424:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829431.816426:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829431.816432:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2867 00000100:00080000:1.0:1787829431.816436:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787829431.816439:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829431/real 1787829431] req@ffff9e35da6f5a40 x1874672085828736/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829456 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829431.816443:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829431.816450:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:0.0:1787829431.816456:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5a40 x1874672085828736/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829456 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829431.816463:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4f00 x1874672085829504/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829431.816467:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829431.816475:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829431.816479:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085828736:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829431.816484:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085828864:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829431.816506:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829431.816518:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829431/real 1787829431] req@ffff9e35da6f5e00 x1874672085828864/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829456 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829431.816533:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5e00 x1874672085828864/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829456 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829431.816542:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829431.816546:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829431.816550:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085828864:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829431.816556:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085828992:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829431.816577:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829431.816587:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829431/real 1787829431] req@ffff9e35da6f4b40 x1874672085828992/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829456 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829431.816595:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4b40 x1874672085828992/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829456 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829431.816601:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829431.816603:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829431.816606:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085828992:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829431.816610:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085829120:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829431.816622:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829431.816628:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829431/real 1787829431] req@ffff9e35da6f70c0 x1874672085829120/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829456 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829431.816636:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f70c0 x1874672085829120/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829456 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829431.816641:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829431.816643:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829431.816645:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085829120:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829431.816663:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085829248:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829431.816674:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829431.816680:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829431/real 1787829431] req@ffff9e35da6f43c0 x1874672085829248/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829456 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829431.816697:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f43c0 x1874672085829248/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829456 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829431.816703:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829431.816705:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829431.816707:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085829248:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829431.816711:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085829376:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829431.816722:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829431.816729:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829431/real 1787829431] req@ffff9e35da6f61c0 x1874672085829376/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829456 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829431.816736:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f61c0 x1874672085829376/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829456 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829431.816742:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829431.816743:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829431.816745:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085829376:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829431.816749:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085829504:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829431.816759:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829431.816765:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829431/real 1787829431] req@ffff9e35da6f4f00 x1874672085829504/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829456 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829431.816773:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4f00 x1874672085829504/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829456 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829431.816778:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829431.816779:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829431.816782:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085829504:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829431.943885:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829431.943889:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829431.943902:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829431.943930:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829432.967933:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829432.967945:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829432.967974:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829432.968036:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829433.991883:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829433.991907:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829433.991912:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829433.991940:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829435.015965:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829435.015969:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829435.015988:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829435.016036:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829436.039890:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829436.039917:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829436.039925:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829436.039993:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829436.935984:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829436.935998:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787829436.936004:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829436.936017:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2872 00000100:00080000:3.0:1787829436.936031:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829436.936040:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787829436.936100:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498f00 x1874672085829632/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829436.936141:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829436.936146:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829436.936148:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829436.936153:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2872 00000100:00080000:3.0:1787829436.936157:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829436.936163:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829436.936176:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829436.936204:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085829632:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1787829436.936209:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499a40 x1874672085829760/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829436.936224:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829436.936228:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787829436.936231:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829436.936236:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2872 00000100:00080000:3.0:1787829436.936241:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829436.936245:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829436.936252:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829436.936267:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498780 x1874672085829888/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829436.936275:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1787829436.936296:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829436.936300:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787829436.936303:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829436.936309:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2872 00000100:00080000:3.0:1787829436.936315:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829436.936322:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787829436.936323:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829436/real 1787829436] req@ffff9e35c6498f00 x1874672085829632/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829461 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1787829436.936332:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:0.0:1787829436.936341:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498f00 x1874672085829632/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829461 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829436.936350:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a940 x1874672085830016/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829436.936351:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829436.936359:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829436.936367:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085829632:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1787829436.936376:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829436.936382:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787829436.936384:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829436.936390:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2872 00000100:00080000:3.0:1787829436.936395:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787829436.936396:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085829760:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1787829436.936403:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829436.936411:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000800:00000100:0.0:1787829436.936418:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829436.936433:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829436/real 1787829436] req@ffff9e35c6499a40 x1874672085829760/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829461 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829436.936434:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498b40 x1874672085830144/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829436.936456:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499a40 x1874672085829760/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829461 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829436.936466:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829436.936467:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829436.936475:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:3.0:1787829436.936475:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829436.936482:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787829436.936483:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085829760:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829436.936495:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085829888:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1787829436.936495:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2872 00000100:00080000:3.0:1787829436.936505:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829436.936513:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829436.936520:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1787829436.936526:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829436.936535:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829436/real 1787829436] req@ffff9e35c6498780 x1874672085829888/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829461 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829436.936540:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649bc00 x1874672085830272/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829436.936555:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498780 x1874672085829888/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829461 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829436.936561:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829436.936565:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829436.936568:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787829436.936570:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:3.0:1787829436.936573:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787829436.936574:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085829888:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1787829436.936584:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2872 00000100:00100000:0.0:1787829436.936588:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085830016:192.168.203.141@tcp:8:kworker.0 00000100:00080000:3.0:1787829436.936591:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829436.936599:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829436.936606:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1787829436.936616:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787829436.936624:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b840 x1874672085830400/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829436.936633:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829436/real 1787829436] req@ffff9e35c649a940 x1874672085830016/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829461 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829436.936647:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a940 x1874672085830016/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829461 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829436.936652:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829436.936654:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829436.936662:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:3.0:1787829436.936662:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:0.0:1787829436.936667:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085830016:192.168.203.141@tcp:8:kworker.0 00000100:00080000:3.0:1787829436.936669:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829436.936677:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2872 00000100:00080000:3.0:1787829436.936683:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787829436.936687:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085830144:192.168.203.141@tcp:8:kworker.0 00000100:00080000:3.0:1787829436.936692:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829436.936701:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000800:00000100:0.0:1787829436.936708:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787829436.936719:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499680 x1874672085830528/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829436.936721:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829436/real 1787829436] req@ffff9e35c6498b40 x1874672085830144/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829461 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829436.936753:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498b40 x1874672085830144/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829461 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829436.936758:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829436.936763:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829436.936766:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829436.936768:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:3.0:1787829436.936771:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787829436.936773:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085830144:192.168.203.141@tcp:8:kworker.0 00000100:00080000:3.0:1787829436.936782:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2872 00000100:00100000:0.0:1787829436.936783:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085830272:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1787829436.936792:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829436.936830:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829436.936844:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:3.0:1787829436.936859:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a940 x1874672085830656/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829436.936869:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829436.936886:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829436/real 1787829436] req@ffff9e35c649bc00 x1874672085830272/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829461 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829436.936898:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649bc00 x1874672085830272/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829461 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829436.936908:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829436.936913:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829436.936917:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085830272:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829436.936925:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085830400:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829436.936947:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829436.936957:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829436/real 1787829436] req@ffff9e35c649b840 x1874672085830400/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829461 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829436.936965:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b840 x1874672085830400/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829461 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829436.936971:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829436.936974:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829436.936976:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085830400:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829436.936987:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085830528:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829436.937001:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829436.937008:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829436/real 1787829436] req@ffff9e35c6499680 x1874672085830528/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829461 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829436.937015:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499680 x1874672085830528/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829461 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829436.937021:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829436.937023:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829436.937025:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085830528:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829436.937030:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085830656:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829436.937039:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829436.937045:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829436/real 1787829436] req@ffff9e35c649a940 x1874672085830656/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829461 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829436.937052:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a940 x1874672085830656/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829461 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829436.937057:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829436.937059:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829436.937062:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085830656:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829437.063930:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829437.063930:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829437.063931:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829437.064001:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829438.087928:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829438.087928:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829438.088001:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829438.088098:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829439.111931:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829439.111931:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829439.111950:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829439.111988:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829440.135867:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829440.135869:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829440.135898:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829440.135945:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829441.159883:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829441.159884:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829441.159904:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829441.159938:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829442.055910:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829442.055921:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787829442.055926:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829442.055937:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2877 00000100:00080000:0.0:1787829442.055948:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829442.055955:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787829442.055995:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7192c0 x1874672085830784/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829442.056024:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829442.056027:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829442.056028:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829442.056032:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2877 00000100:00080000:0.0:1787829442.056034:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829442.056036:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829442.056046:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829442.056058:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a1c0 x1874672085830912/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829442.056068:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829442.056071:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787829442.056073:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829442.056076:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2877 00000100:00080000:0.0:1787829442.056078:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829442.056080:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829442.056083:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829442.056091:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718b40 x1874672085831040/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829442.056099:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829442.056105:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829442.056107:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787829442.056107:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085830784:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1787829442.056118:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2877 00000100:00080000:0.0:1787829442.056121:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829442.056125:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829442.056128:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:0.0:1787829442.056139:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718000 x1874672085831168/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829442.056160:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829442.056164:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829442.056166:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829442.056171:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2877 00000100:00080000:0.0:1787829442.056195:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829442.056198:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829442.056202:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:0.0:1787829442.056216:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a580 x1874672085831296/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787829442.056221:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787829442.056232:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829442.056236:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829442.056238:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829442.056245:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2877 00000100:00080000:0.0:1787829442.056249:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829442.056252:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829442.056255:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829442.056266:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da719a40 x1874672085831424/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829442.056274:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829442.056277:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787829442.056279:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829442.056284:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2877 00000100:00100000:1.0:1787829442.056286:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829442/real 1787829442] req@ffff9e35da7192c0 x1874672085830784/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829467 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829442.056289:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829442.056300:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829442.056313:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829442.056331:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7183c0 x1874672085831552/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829442.056339:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787829442.056341:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7192c0 x1874672085830784/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829467 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829442.056343:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829442.056348:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829442.056355:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2877 00000100:00080000:0.0:1787829442.056360:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829442.056368:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829442.056372:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00080000:1.0:1787829442.056385:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1787829442.056386:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a940 x1874672085831680/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829442.056405:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829442.056405:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:0.0:1787829442.056420:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829442.056424:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787829442.056426:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085830784:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1787829442.056430:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2877 00000100:00080000:0.0:1787829442.056435:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829442.056444:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829442.056450:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:1.0:1787829442.056456:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085830912:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829442.056465:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718f00 x1874672085831808/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787829442.056494:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829442.056515:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829442/real 1787829442] req@ffff9e35da71a1c0 x1874672085830912/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829467 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829442.056530:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a1c0 x1874672085830912/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829467 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829442.056540:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829442.056544:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829442.056547:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085830912:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829442.056575:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085831040:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787829442.056594:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829442.056605:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829442/real 1787829442] req@ffff9e35da718b40 x1874672085831040/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829467 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829442.056618:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718b40 x1874672085831040/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829467 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829442.056627:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829442.056630:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829442.056634:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085831040:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829442.056640:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085831168:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829442.056657:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829442.056666:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829442/real 1787829442] req@ffff9e35da718000 x1874672085831168/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829467 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829442.056678:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718000 x1874672085831168/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829467 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829442.056686:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829442.056689:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829442.056693:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085831168:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829442.056699:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085831296:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829442.056714:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829442.056722:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829442/real 1787829442] req@ffff9e35da71a580 x1874672085831296/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829467 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829442.056733:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672085831296/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829467 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829442.056742:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829442.056745:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829442.056748:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085831296:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829442.056754:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085831424:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787829442.056772:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829442.056782:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829442/real 1787829442] req@ffff9e35da719a40 x1874672085831424/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829467 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829442.056794:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672085831424/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829467 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829442.056852:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829442.056857:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829442.056860:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085831424:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829442.056868:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085831552:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787829442.056891:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829442.056905:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829442/real 1787829442] req@ffff9e35da7183c0 x1874672085831552/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829467 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829442.056918:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672085831552/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829467 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829442.056928:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829442.056931:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829442.056935:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085831552:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829442.056942:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085831680:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829442.056962:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829442.056974:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829442/real 1787829442] req@ffff9e35da71a940 x1874672085831680/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829467 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829442.056986:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672085831680/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829467 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829442.056995:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829442.056999:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829442.057002:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085831680:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829442.057009:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085831808:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829442.057027:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829442.057037:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829442/real 1787829442] req@ffff9e35da718f00 x1874672085831808/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829467 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829442.057050:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672085831808/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829467 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829442.057059:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829442.057063:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829442.057066:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085831808:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829442.183935:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829442.183945:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829442.183956:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:0.0:1787829442.184007:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829443.207979:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829443.207988:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829443.207991:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829443.208049:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829444.231952:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829444.231953:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829444.231964:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829444.232048:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829445.255903:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829445.255906:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829445.255906:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:0.0:1787829445.255986:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829446.279968:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829446.279972:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829446.280020:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829446.280180:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00080000:1.0:1787829447.175962:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829447.175973:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829447.175977:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829447.175988:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2882 00000100:00080000:1.0:1787829447.176000:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829447.176008:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787829447.176048:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6580 x1874672085831936/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829447.176083:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829447.176087:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829447.176090:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829447.176096:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2882 00000100:00080000:1.0:1787829447.176100:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829447.176105:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829447.176132:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829447.176143:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085831936:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1787829447.176153:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7480 x1874672085832064/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829447.176179:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829447.176183:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829447.176185:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829447.176192:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2882 00000100:00080000:1.0:1787829447.176199:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829447.176203:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829447.176209:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829447.176222:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4000 x1874672085832192/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829447.176235:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787829447.176242:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829447.176247:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829447.176251:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829447.176259:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2882 00000100:00080000:1.0:1787829447.176264:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829447.176269:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829447.176276:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:0.0:1787829447.176289:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829447/real 1787829447] req@ffff9e35da6f6580 x1874672085831936/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829472 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829447.176295:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6940 x1874672085832320/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829447.176321:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829447.176327:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:0.0:1787829447.176333:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6580 x1874672085831936/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829472 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829447.176333:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829447.176348:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2882 00000100:00080000:1.0:1787829447.176361:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829447.176363:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829447.176370:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:1.0:1787829447.176370:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787829447.176382:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085831936:192.168.203.141@tcp:250:kworker.0 00000008:00100000:1.0:1787829447.176383:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:1.0:1787829447.176406:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6d00 x1874672085832448/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829447.176415:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085832064:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787829447.176431:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829447.176439:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829447.176442:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1787829447.176448:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787829447.176454:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2882 00000100:00080000:1.0:1787829447.176461:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787829447.176463:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829447/real 1787829447] req@ffff9e35da6f7480 x1874672085832064/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829472 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829447.176470:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829447.176484:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829447.176486:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7480 x1874672085832064/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829472 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829447.176503:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829447.176513:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829447.176516:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4780 x1874672085832576/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829447.176520:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085832064:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829447.176539:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085832192:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787829447.176561:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829447.176566:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829447.176569:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1787829447.176578:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787829447.176583:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2882 00000100:00080000:1.0:1787829447.176591:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829447.176597:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787829447.176603:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829447/real 1787829447] req@ffff9e35da6f4000 x1874672085832192/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829472 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1787829447.176605:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829447.176631:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6580 x1874672085832704/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829447.176634:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4000 x1874672085832192/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829472 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829447.176718:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829447.176726:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:1.0:1787829447.176727:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787829447.176736:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085832192:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787829447.176738:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829447.176744:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787829447.176747:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085832320:192.168.203.141@tcp:8:kworker.0 00000100:00080000:1.0:1787829447.176756:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2882 00000100:00080000:1.0:1787829447.176762:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829447.176766:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829447.176772:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000800:00000100:0.0:1787829447.176776:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829447.176791:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829447/real 1787829447] req@ffff9e35da6f6940 x1874672085832320/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829472 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829447.176829:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6940 x1874672085832320/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829472 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829447.176837:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829447.176840:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829447.176842:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085832320:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829447.176849:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4f00 x1874672085832832/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829447.176864:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085832448:192.168.203.141@tcp:8:kworker.0 00000100:00080000:1.0:1787829447.176868:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829447.176873:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829447.176877:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829447.176886:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2882 00000800:00000100:0.0:1787829447.176887:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787829447.176899:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787829447.176906:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829447/real 1787829447] req@ffff9e35da6f6d00 x1874672085832448/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829472 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829447.176908:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829447.176923:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:0.0:1787829447.176929:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6d00 x1874672085832448/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829472 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829447.176939:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829447.176941:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829447.176947:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085832448:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829447.176949:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f61c0 x1874672085832960/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829447.176957:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085832576:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829447.176988:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829447.176999:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829447/real 1787829447] req@ffff9e35da6f4780 x1874672085832576/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829472 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829447.177013:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4780 x1874672085832576/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829472 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829447.177019:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829447.177022:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829447.177024:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085832576:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829447.177028:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085832704:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829447.177039:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829447.177045:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829447/real 1787829447] req@ffff9e35da6f6580 x1874672085832704/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829472 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829447.177052:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6580 x1874672085832704/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829472 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829447.177057:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829447.177059:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829447.177061:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085832704:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829447.177071:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085832832:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829447.177081:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829447.177088:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829447/real 1787829447] req@ffff9e35da6f4f00 x1874672085832832/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829472 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829447.177096:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4f00 x1874672085832832/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829472 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829447.177102:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829447.177103:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829447.177105:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085832832:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829447.177124:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085832960:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829447.177140:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829447.177148:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829447/real 1787829447] req@ffff9e35da6f61c0 x1874672085832960/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829472 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829447.177155:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f61c0 x1874672085832960/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829472 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829447.177161:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829447.177164:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829447.177166:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085832960:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829447.303916:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829447.303954:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829447.303960:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829447.304000:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829448.327942:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829448.327959:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829448.328000:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:0.0:1787829448.328047:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829449.351917:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829449.351949:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829449.351976:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829449.351979:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829450.375907:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829450.375928:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829450.375931:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787829450.375955:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829451.399887:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829451.399898:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829451.399900:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787829451.399930:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829452.296325:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829452.296336:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787829452.296341:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829452.296352:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2887 00000100:00080000:3.0:1787829452.296364:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829452.296373:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787829452.296412:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498780 x1874672085833088/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829452.296446:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829452.296450:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829452.296452:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829452.296457:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2887 00000100:00080000:3.0:1787829452.296462:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829452.296466:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829452.296488:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829452.296490:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085833088:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1787829452.296509:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499a40 x1874672085833216/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829452.296522:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829452.296526:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787829452.296527:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829452.296533:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2887 00000100:00080000:3.0:1787829452.296537:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829452.296541:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829452.296548:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829452.296562:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498f00 x1874672085833344/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829452.296573:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829452.296576:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787829452.296579:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829452.296584:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2887 00000100:00080000:3.0:1787829452.296588:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829452.296594:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829452.296600:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:0.0:1787829452.296609:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085833216:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787829452.296617:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a1c0 x1874672085833472/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829452.296632:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829452.296636:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787829452.296638:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787829452.296641:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085833344:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1787829452.296647:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2887 00000100:00080000:3.0:1787829452.296653:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829452.296657:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829452.296664:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:0.0:1787829452.296665:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085833472:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787829452.296686:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a580 x1874672085833600/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829452.296702:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829452.296706:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829452.296709:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829452.296714:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2887 00000100:00100000:0.0:1787829452.296720:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085833600:192.168.203.141@tcp:8:kworker.0 00000100:00080000:3.0:1787829452.296720:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829452.296731:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829452.296738:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829452.296756:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b0c0 x1874672085833728/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829452.296777:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829452.296781:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787829452.296783:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829452.296796:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2887 00000100:00100000:0.0:1787829452.296826:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085833728:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1787829452.296843:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829452.296851:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829452.296858:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829452.296873:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b480 x1874672085833856/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829452.296903:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829452.296907:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787829452.296909:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829452.296914:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2887 00000100:00080000:3.0:1787829452.296918:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829452.296922:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829452.296927:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:3.0:1787829452.296940:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c64992c0 x1874672085833984/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829452.296949:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829452.296952:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787829452.296954:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829452.296960:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2887 00000100:00080000:3.0:1787829452.296963:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829452.296967:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829452.296972:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:3.0:1787829452.296982:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499e00 x1874672085834112/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829452.297412:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085833856:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829452.297451:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c64992c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085833984:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829452.297473:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085834112:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829452.423907:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829452.423907:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829452.423924:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787829452.423957:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829453.447873:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829453.447880:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829453.447884:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787829453.447976:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829454.471913:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829454.471945:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829454.471962:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787829454.471990:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1787829455.368072:0:2399:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.141:988 00000400:00000100:2.0:1787829455.368115:0:2399:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.141@tcp at host 192.168.203.141 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1787829455.368134:0:2399:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:2.0:1787829455.368185:0:2399:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000800:00000100:2.0:1787829455.368209:0:2399:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:1.0:1787829455.368217:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829452/real 1787829455] req@ffff9e35c6499e00 x1874672085834112/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829477 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:2.0:1787829455.368218:0:2399:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:2.0:1787829455.368233:0:2399:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:2.0:1787829455.368255:0:2399:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:1.0:1787829455.368262:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499e00 x1874672085834112/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829477 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:2.0:1787829455.368274:0:2399:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00080000:1.0:1787829455.368290:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000800:00000100:2.0:1787829455.368296:0:2399:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00080000:1.0:1787829455.368302:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829455.368310:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085834112:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829455.368320:0:2399:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:1.0:1787829455.368325:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829452/real 1787829455] req@ffff9e35c649b0c0 x1874672085833728/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829477 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829455.368344:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b0c0 x1874672085833728/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829477 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829455.368354:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829455.368358:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829455.368362:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085833728:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829455.368369:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829452/real 1787829455] req@ffff9e35c649a580 x1874672085833600/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829477 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829455.368390:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a580 x1874672085833600/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829477 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829455.368400:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829455.368403:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829455.368406:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085833600:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829455.368411:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829452/real 1787829455] req@ffff9e35c649a1c0 x1874672085833472/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829477 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829455.368419:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a1c0 x1874672085833472/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829477 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829455.368428:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829455.368432:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829455.368436:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085833472:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829455.368444:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829452/real 1787829455] req@ffff9e35c6499a40 x1874672085833216/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829477 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829455.368458:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499a40 x1874672085833216/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829477 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829455.368467:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829455.368471:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829455.368474:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085833216:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829455.368482:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829452/real 1787829455] req@ffff9e35c6498f00 x1874672085833344/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829477 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829455.368496:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498f00 x1874672085833344/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829477 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829455.368505:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829455.368509:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829455.368512:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085833344:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829455.368521:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829452/real 1787829455] req@ffff9e35c6498780 x1874672085833088/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829477 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829455.368535:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498780 x1874672085833088/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829477 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829455.368545:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829455.368548:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829455.368553:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085833088:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1787829455.368597:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829452/real 1787829455] req@ffff9e35c649b480 x1874672085833856/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829477 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829455.368613:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b480 x1874672085833856/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829477 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829455.368621:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829455.368625:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829455.368628:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085833856:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829455.368635:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829452/real 1787829455] req@ffff9e35c64992c0 x1874672085833984/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829477 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829455.368648:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c64992c0 x1874672085833984/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829477 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829455.368657:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829455.368661:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829455.368665:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c64992c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085833984:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829455.495904:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829455.495911:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829455.495941:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:1.0:1787829455.496010:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829456.519920:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829456.519949:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829456.519950:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787829456.519963:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829457.415952:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829457.415963:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787829457.415968:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829457.415979:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2892 00000100:00080000:0.0:1787829457.415990:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829457.415998:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787829457.416037:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718780 x1874672085834240/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829457.416080:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829457.416084:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829457.416086:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829457.416091:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2892 00000100:00080000:0.0:1787829457.416096:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829457.416103:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829457.416117:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829457.416135:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085834240:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1787829457.416137:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718f00 x1874672085834368/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829457.416155:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829457.416159:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787829457.416161:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829457.416166:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2892 00000100:00080000:0.0:1787829457.416170:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829457.416190:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829457.416196:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829457.416211:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a940 x1874672085834496/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787829457.416224:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787829457.416225:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829457.416232:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829457.416234:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829457.416240:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2892 00000100:00080000:0.0:1787829457.416246:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829457.416252:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829457.416259:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:0.0:1787829457.416272:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7183c0 x1874672085834624/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829457.416283:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829457.416289:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829457.416291:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787829457.416295:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829457/real 1787829457] req@ffff9e35da718780 x1874672085834240/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829482 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829457.416300:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2892 00000100:00080000:0.0:1787829457.416308:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829457.416316:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829457.416323:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:1.0:1787829457.416327:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718780 x1874672085834240/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829482 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829457.416341:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da719a40 x1874672085834752/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829457.416344:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829457.416354:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:0.0:1787829457.416361:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787829457.416363:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085834240:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1787829457.416370:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829457.416374:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829457.416379:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2892 00000100:00080000:0.0:1787829457.416384:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829457.416389:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829457.416394:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829457.416402:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085834368:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829457.416410:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a580 x1874672085834880/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829457.416422:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829457.416426:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787829457.416428:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829457.416432:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2892 00000800:00000100:1.0:1787829457.416438:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787829457.416440:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829457.416447:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829457.416455:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829457.416460:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829457/real 1787829457] req@ffff9e35da718f00 x1874672085834368/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829482 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829457.416471:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718000 x1874672085835008/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829457.416485:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672085834368/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829482 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829457.416487:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829457.416498:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829457.416503:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829457.416508:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829457.416516:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:0.0:1787829457.416520:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2892 00000100:00100000:1.0:1787829457.416526:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085834368:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787829457.416528:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829457.416537:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787829457.416541:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085834496:192.168.203.141@tcp:38:kworker.0 00000008:00100000:0.0:1787829457.416545:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:0.0:1787829457.416570:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718b40 x1874672085835136/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787829457.416577:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787829457.416589:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829457.416593:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:1.0:1787829457.416596:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829457/real 1787829457] req@ffff9e35da71a940 x1874672085834496/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829482 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829457.416597:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829457.416607:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2892 00000100:00080000:0.0:1787829457.416615:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829457.416623:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787829457.416623:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672085834496/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829482 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1787829457.416633:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00080000:1.0:1787829457.416637:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829457.416644:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829457.416651:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a1c0 x1874672085835264/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829457.416652:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085834496:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829457.416666:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085834624:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829457.416693:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829457.416705:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829457/real 1787829457] req@ffff9e35da7183c0 x1874672085834624/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829482 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829457.416717:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672085834624/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829482 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829457.416731:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829457.416735:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829457.416738:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085834624:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829457.416744:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085834752:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829457.416764:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829457.416775:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829457/real 1787829457] req@ffff9e35da719a40 x1874672085834752/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829482 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829457.416788:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672085834752/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829482 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829457.416837:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829457.416842:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829457.416846:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085834752:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829457.416869:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085834880:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787829457.416893:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829457.416912:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829457/real 1787829457] req@ffff9e35da71a580 x1874672085834880/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829482 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829457.416930:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672085834880/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829482 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829457.416939:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829457.416942:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829457.416946:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085834880:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829457.416952:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085835008:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787829457.416974:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829457.416986:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829457/real 1787829457] req@ffff9e35da718000 x1874672085835008/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829482 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829457.416996:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718000 x1874672085835008/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829482 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829457.417003:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829457.417007:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829457.417010:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085835008:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829457.417017:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085835136:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829457.417056:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829457.417069:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829457/real 1787829457] req@ffff9e35da718b40 x1874672085835136/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829482 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829457.417081:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718b40 x1874672085835136/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829482 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829457.417090:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829457.417094:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829457.417097:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085835136:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829457.417103:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085835264:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829457.417122:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829457.417135:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829457/real 1787829457] req@ffff9e35da71a1c0 x1874672085835264/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829482 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829457.417146:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a1c0 x1874672085835264/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829482 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829457.417161:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829457.417165:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829457.417169:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085835264:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829457.543878:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829457.543927:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829457.543947:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829457.543962:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829458.567922:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829458.567959:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829458.567990:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787829458.568045:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829459.591902:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829459.591932:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829459.591952:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829459.591960:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829460.615922:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829460.615925:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829460.615948:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829460.615967:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829461.639907:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829461.639909:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787829461.639953:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829461.639958:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787829462.535916:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829462.535923:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829462.535927:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829462.535935:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2897 00000100:00080000:1.0:1787829462.535949:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829462.535955:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787829462.535987:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f43c0 x1874672085835392/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829462.536032:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829462.536037:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829462.536040:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829462.536044:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2897 00000100:00080000:1.0:1787829462.536049:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829462.536055:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829462.536066:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829462.536074:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085835392:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1787829462.536081:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f70c0 x1874672085835520/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829462.536092:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829462.536095:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829462.536097:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829462.536101:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2897 00000100:00080000:1.0:1787829462.536105:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829462.536109:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829462.536112:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829462.536121:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4b40 x1874672085835648/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829462.536129:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829462.536131:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829462.536133:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829462.536135:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2897 00000100:00080000:1.0:1787829462.536138:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1787829462.536139:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787829462.536143:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829462.536147:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:1.0:1787829462.536156:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5e00 x1874672085835776/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829462.536167:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829462.536169:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829462.536171:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829462.536174:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2897 00000100:00080000:1.0:1787829462.536176:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829462.536179:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829462.536182:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:2.0:1787829462.536188:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829462/real 1787829462] req@ffff9e35da6f43c0 x1874672085835392/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829487 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829462.536192:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5a40 x1874672085835904/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829462.536216:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829462.536223:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829462.536226:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829462.536232:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2897 00000100:00080000:1.0:1787829462.536237:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829462.536242:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829462.536245:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829462.536252:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7840 x1874672085836032/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829462.536259:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f43c0 x1874672085835392/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829487 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829462.536261:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829462.536267:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829462.536269:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829462.536277:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2897 00000100:00080000:1.0:1787829462.536283:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829462.536286:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:2.0:1787829462.536286:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:1.0:1787829462.536293:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1787829462.536296:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829462.536301:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085835392:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1787829462.536305:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f61c0 x1874672085836160/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829462.536313:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829462.536315:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829462.536316:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829462.536319:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2897 00000100:00080000:1.0:1787829462.536321:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829462.536324:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829462.536327:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:2.0:1787829462.536329:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085835520:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829462.536336:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4f00 x1874672085836288/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829462.536344:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829462.536347:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829462.536348:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829462.536351:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2897 00000100:00080000:1.0:1787829462.536354:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1787829462.536359:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787829462.536360:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829462.536366:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:1.0:1787829462.536375:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6580 x1874672085836416/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829462.536383:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829462/real 1787829462] req@ffff9e35da6f70c0 x1874672085835520/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829487 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829462.536406:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f70c0 x1874672085835520/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829487 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829462.536417:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829462.536421:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829462.536425:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085835520:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829462.536433:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085835648:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787829462.536472:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829462.536490:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829462/real 1787829462] req@ffff9e35da6f4b40 x1874672085835648/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829487 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829462.536508:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4b40 x1874672085835648/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829487 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829462.536527:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829462.536531:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829462.536535:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085835648:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829462.536541:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085835776:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829462.536564:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829462.536576:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829462/real 1787829462] req@ffff9e35da6f5e00 x1874672085835776/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829487 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829462.536585:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5e00 x1874672085835776/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829487 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829462.536591:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829462.536593:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829462.536595:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085835776:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829462.536599:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085835904:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829462.536612:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829462.536619:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829462/real 1787829462] req@ffff9e35da6f5a40 x1874672085835904/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829487 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829462.536626:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5a40 x1874672085835904/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829487 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829462.536631:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829462.536633:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829462.536635:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085835904:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829462.536639:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085836032:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787829462.536654:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829462.536665:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829462/real 1787829462] req@ffff9e35da6f7840 x1874672085836032/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829487 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829462.536677:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7840 x1874672085836032/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829487 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829462.536686:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829462.536689:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829462.536693:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085836032:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829462.536699:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085836160:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787829462.536720:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829462.536732:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829462/real 1787829462] req@ffff9e35da6f61c0 x1874672085836160/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829487 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829462.536746:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f61c0 x1874672085836160/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829487 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829462.536755:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829462.536758:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829462.536761:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085836160:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829462.536793:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085836288:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829462.536847:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829462.536860:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829462/real 1787829462] req@ffff9e35da6f4f00 x1874672085836288/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829487 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829462.536869:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4f00 x1874672085836288/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829487 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829462.536875:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829462.536877:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829462.536879:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085836288:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829462.536883:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085836416:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829462.536895:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829462.536901:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829462/real 1787829462] req@ffff9e35da6f6580 x1874672085836416/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829487 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829462.536909:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6580 x1874672085836416/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829487 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829462.536914:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829462.536916:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829462.536918:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085836416:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829462.663966:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829462.663966:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829462.664020:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829462.664054:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829463.687936:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829463.687936:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829463.687979:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829463.687992:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829464.711898:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829464.711900:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829464.711927:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787829464.711948:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829465.735916:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829465.735916:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829465.735932:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787829465.735962:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829466.759941:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829466.759941:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829466.760004:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829466.760027:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787829467.655889:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829467.655900:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829467.655904:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829467.655915:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2903 00000100:00080000:1.0:1787829467.655926:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829467.655933:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787829467.655989:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4780 x1874672085836544/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829467.656016:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829467.656019:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829467.656021:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829467.656024:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2903 00000100:00080000:1.0:1787829467.656027:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829467.656030:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829467.656039:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829467.656051:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6d00 x1874672085836672/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829467.656059:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829467.656062:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829467.656065:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829467.656068:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2903 00000100:00080000:1.0:1787829467.656070:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829467.656073:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829467.656077:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829467.656085:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6940 x1874672085836800/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829467.656098:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787829467.656098:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085836544:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787829467.656105:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829467.656110:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829467.656124:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2903 00000100:00080000:1.0:1787829467.656127:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829467.656130:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829467.656133:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:1.0:1787829467.656145:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4000 x1874672085836928/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829467.656151:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829467.656155:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829467.656156:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829467.656159:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2903 00000100:00080000:1.0:1787829467.656169:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829467.656173:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829467.656177:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:1.0:1787829467.656189:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7480 x1874672085837056/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829467.656190:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787829467.656203:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829467.656206:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829467.656209:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829467.656214:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2903 00000100:00080000:1.0:1787829467.656220:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829467.656222:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829467.656225:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829467.656235:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6580 x1874672085837184/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829467.656242:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829467.656245:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829467.656248:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829467.656252:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2903 00000100:00100000:2.0:1787829467.656253:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829467/real 1787829467] req@ffff9e35da6f4780 x1874672085836544/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829492 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829467.656255:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829467.656267:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829467.656282:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829467.656289:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4780 x1874672085836544/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829492 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829467.656303:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1787829467.656309:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4f00 x1874672085837312/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829467.656310:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829467.656322:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085836544:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1787829467.656345:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085836672:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787829467.656347:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829467.656354:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829467.656356:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829467.656370:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2903 00000800:00000100:2.0:1787829467.656379:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787829467.656382:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829467.656388:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829467.656394:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:2.0:1787829467.656406:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829467/real 1787829467] req@ffff9e35da6f6d00 x1874672085836672/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829492 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829467.656409:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f61c0 x1874672085837440/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829467.656441:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829467.656455:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829467.656457:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829467.656462:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2903 00000100:00080000:1.0:1787829467.656466:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829467.656473:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787829467.656475:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6d00 x1874672085836672/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829492 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1787829467.656477:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00080000:2.0:1787829467.656499:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1787829467.656500:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7840 x1874672085837568/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829467.656508:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829467.656512:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085836672:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829467.656541:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085836800:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787829467.656573:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829467.656586:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829467/real 1787829467] req@ffff9e35da6f6940 x1874672085836800/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829492 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829467.656606:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6940 x1874672085836800/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829492 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829467.656612:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829467.656614:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829467.656617:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085836800:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829467.656621:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085836928:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829467.656638:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829467.656645:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829467/real 1787829467] req@ffff9e35da6f4000 x1874672085836928/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829492 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829467.656653:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4000 x1874672085836928/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829492 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829467.656660:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829467.656663:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829467.656667:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085836928:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829467.656674:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085837056:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829467.656695:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829467.656707:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829467/real 1787829467] req@ffff9e35da6f7480 x1874672085837056/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829492 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829467.656720:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7480 x1874672085837056/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829492 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829467.656731:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829467.656733:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829467.656735:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085837056:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829467.656740:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085837184:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787829467.656755:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829467.656763:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829467/real 1787829467] req@ffff9e35da6f6580 x1874672085837184/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829492 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829467.656771:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6580 x1874672085837184/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829492 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829467.656776:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829467.656779:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829467.656781:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085837184:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829467.656784:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085837312:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787829467.656794:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829467.656842:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829467/real 1787829467] req@ffff9e35da6f4f00 x1874672085837312/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829492 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829467.656850:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4f00 x1874672085837312/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829492 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829467.656856:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829467.656858:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829467.656860:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085837312:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829467.656865:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085837440:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829467.656878:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829467.656885:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829467/real 1787829467] req@ffff9e35da6f61c0 x1874672085837440/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829492 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829467.656892:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f61c0 x1874672085837440/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829492 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829467.656897:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829467.656899:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829467.656901:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085837440:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829467.656905:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085837568:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829467.656914:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829467.656920:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829467/real 1787829467] req@ffff9e35da6f7840 x1874672085837568/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829492 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829467.656928:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7840 x1874672085837568/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829492 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829467.656933:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829467.656935:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829467.656936:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085837568:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829467.783939:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829467.783939:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829467.784015:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829467.784017:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829468.807898:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829468.807898:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829468.807958:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787829468.808028:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829469.831937:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829469.831937:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829469.831991:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829469.832006:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829470.855905:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829470.855905:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829470.855907:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829470.855993:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829471.879972:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829471.879972:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829471.879984:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787829471.880107:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829472.775967:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829472.775980:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787829472.775986:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829472.775999:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2908 00000100:00080000:0.0:1787829472.776013:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829472.776021:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787829472.776076:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7192c0 x1874672085837696/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829472.776109:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829472.776114:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829472.776116:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829472.776121:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2908 00000100:00080000:0.0:1787829472.776126:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829472.776132:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829472.776145:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829472.776156:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085837696:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1787829472.776167:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a1c0 x1874672085837824/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829472.776180:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829472.776184:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787829472.776186:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829472.776192:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2908 00000100:00080000:0.0:1787829472.776197:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829472.776201:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829472.776207:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829472.776221:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718b40 x1874672085837952/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829472.776229:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829472.776232:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829472.776233:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829472.776239:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2908 00000100:00080000:0.0:1787829472.776243:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829472.776245:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:2.0:1787829472.776249:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:0.0:1787829472.776251:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:0.0:1787829472.776261:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718000 x1874672085838080/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829472.776269:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829472.776272:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829472.776274:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829472.776277:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2908 00000100:00080000:0.0:1787829472.776279:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829472.776281:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829472.776284:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:0.0:1787829472.776292:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a580 x1874672085838208/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829472.776298:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829472.776302:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:2.0:1787829472.776304:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829472/real 1787829472] req@ffff9e35da7192c0 x1874672085837696/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829497 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829472.776305:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829472.776313:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2908 00000100:00080000:0.0:1787829472.776318:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829472.776323:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829472.776326:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829472.776336:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da719a40 x1874672085838336/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829472.776336:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7192c0 x1874672085837696/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829497 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829472.776357:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829472.776363:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787829472.776364:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829472.776367:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829472.776368:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2908 00000100:00080000:0.0:1787829472.776373:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829472.776379:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:2.0:1787829472.776379:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000008:00100000:0.0:1787829472.776385:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829472.776391:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085837696:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1787829472.776395:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7183c0 x1874672085838464/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829472.776406:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829472.776408:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829472.776410:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829472.776413:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2908 00000100:00080000:0.0:1787829472.776415:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829472.776418:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829472.776420:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:0.0:1787829472.776427:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a940 x1874672085838592/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829472.776432:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085837824:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787829472.776436:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829472.776441:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829472.776442:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829472.776445:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2908 00000100:00080000:0.0:1787829472.776448:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829472.776450:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829472.776453:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:0.0:1787829472.776463:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718f00 x1874672085838720/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829472.776474:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829472.776492:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829472/real 1787829472] req@ffff9e35da71a1c0 x1874672085837824/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829497 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829472.776508:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a1c0 x1874672085837824/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829497 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829472.776520:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829472.776524:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829472.776527:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085837824:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829472.776532:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085837952:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787829472.776554:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829472.776578:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829472/real 1787829472] req@ffff9e35da718b40 x1874672085837952/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829497 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829472.776593:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718b40 x1874672085837952/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829497 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829472.776603:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829472.776606:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829472.776610:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085837952:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829472.776617:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085838080:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829472.776639:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829472.776649:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829472/real 1787829472] req@ffff9e35da718000 x1874672085838080/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829497 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829472.776662:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718000 x1874672085838080/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829497 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829472.776672:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829472.776676:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829472.776679:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085838080:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829472.776686:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085838208:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829472.776706:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829472.776714:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829472/real 1787829472] req@ffff9e35da71a580 x1874672085838208/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829497 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829472.776723:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672085838208/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829497 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829472.776730:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829472.776732:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829472.776734:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085838208:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829472.776739:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085838336:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787829472.776750:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829472.776757:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829472/real 1787829472] req@ffff9e35da719a40 x1874672085838336/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829497 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829472.776764:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672085838336/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829497 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829472.776770:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829472.776771:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829472.776773:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085838336:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829472.776777:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085838464:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787829472.776787:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829472.776794:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829472/real 1787829472] req@ffff9e35da7183c0 x1874672085838464/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829497 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829472.776846:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672085838464/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829497 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829472.776852:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829472.776854:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829472.776857:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085838464:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829472.776876:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085838592:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829472.776892:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829472.776900:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829472/real 1787829472] req@ffff9e35da71a940 x1874672085838592/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829497 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829472.776908:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672085838592/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829497 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829472.776914:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829472.776916:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829472.776918:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085838592:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829472.776921:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085838720:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829472.776945:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829472.776957:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829472/real 1787829472] req@ffff9e35da718f00 x1874672085838720/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829497 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829472.776966:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672085838720/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829497 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829472.776972:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829472.776974:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829472.776976:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085838720:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829472.903865:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829472.903891:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829472.903906:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829472.904007:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829473.927898:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829473.927939:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829473.927940:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829473.927983:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829474.951873:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829474.951877:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829474.951944:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:0.0:1787829474.951974:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829475.975941:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829475.975944:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829475.975973:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829475.975999:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829476.999938:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829476.999938:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829476.999938:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829476.999979:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829477.895978:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829477.895989:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829477.895994:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829477.896005:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2913 00000100:00080000:2.0:1787829477.896016:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829477.896022:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787829477.896066:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c3c0 x1874672085838848/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829477.896098:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829477.896101:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829477.896106:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829477.896110:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2913 00000100:00080000:2.0:1787829477.896112:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829477.896115:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829477.896126:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829477.896135:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c780 x1874672085838976/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829477.896143:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829477.896146:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829477.896148:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829477.896151:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2913 00000100:00080000:2.0:1787829477.896154:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829477.896156:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829477.896161:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829477.896165:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085838848:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1787829477.896174:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498de00 x1874672085839104/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829477.896183:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829477.896186:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829477.896187:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829477.896191:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2913 00000100:00080000:2.0:1787829477.896193:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829477.896196:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829477.896199:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:2.0:1787829477.896208:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498ed00 x1874672085839232/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829477.896213:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829477.896216:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829477.896217:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829477.896220:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2913 00000100:00080000:2.0:1787829477.896223:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829477.896225:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829477.896228:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:2.0:1787829477.896236:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f840 x1874672085839360/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829477.896241:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829477.896244:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829477.896246:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829477.896249:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2913 00000100:00080000:2.0:1787829477.896252:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829477.896256:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:1.0:1787829477.896261:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:2.0:1787829477.896261:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829477.896272:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cf00 x1874672085839488/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829477.896280:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829477.896283:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829477.896284:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829477.896287:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2913 00000100:00080000:2.0:1787829477.896289:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829477.896292:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829477.896295:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829477.896303:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498d680 x1874672085839616/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829477.896309:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829477.896312:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829477.896315:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787829477.896317:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829477/real 1787829477] req@ffff9e35c498c3c0 x1874672085838848/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829502 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829477.896321:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2913 00000100:00080000:2.0:1787829477.896325:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829477.896333:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829477.896341:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:1.0:1787829477.896351:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c3c0 x1874672085838848/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829502 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829477.896366:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:2.0:1787829477.896369:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e1c0 x1874672085839744/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829477.896374:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829477.896385:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085838848:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1787829477.896389:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829477.896395:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829477.896398:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787829477.896401:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085838976:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1787829477.896407:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2913 00000100:00080000:2.0:1787829477.896413:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829477.896418:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829477.896425:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000800:00000100:1.0:1787829477.896432:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829477.896443:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f480 x1874672085839872/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829477.896449:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829477/real 1787829477] req@ffff9e35c498c780 x1874672085838976/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829502 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829477.896477:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c780 x1874672085838976/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829502 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829477.896488:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829477.896493:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829477.896497:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085838976:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829477.896537:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085839104:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787829477.896555:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829477.896565:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829477/real 1787829477] req@ffff9e35c498de00 x1874672085839104/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829502 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829477.896587:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498de00 x1874672085839104/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829502 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829477.896597:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829477.896601:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829477.896605:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085839104:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829477.896612:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085839232:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829477.896635:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829477.896646:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829477/real 1787829477] req@ffff9e35c498ed00 x1874672085839232/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829502 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829477.896655:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498ed00 x1874672085839232/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829502 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829477.896661:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829477.896664:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829477.896666:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085839232:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829477.896671:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085839360:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829477.896683:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829477.896689:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829477/real 1787829477] req@ffff9e35c498f840 x1874672085839360/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829502 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829477.896699:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f840 x1874672085839360/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829502 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829477.896708:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829477.896712:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829477.896717:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085839360:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829477.896723:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085839488:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787829477.896745:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829477.896757:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829477/real 1787829477] req@ffff9e35c498cf00 x1874672085839488/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829502 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829477.896765:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498cf00 x1874672085839488/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829502 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829477.896772:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829477.896774:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829477.896776:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085839488:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829477.896781:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085839616:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787829477.896794:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829477.896834:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829477/real 1787829477] req@ffff9e35c498d680 x1874672085839616/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829502 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829477.896847:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498d680 x1874672085839616/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829502 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829477.896856:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829477.896860:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829477.896864:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085839616:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829477.896871:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085839744:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829477.896894:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829477.896925:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829477/real 1787829477] req@ffff9e35c498e1c0 x1874672085839744/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829502 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829477.896939:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e1c0 x1874672085839744/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829502 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829477.896946:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829477.896948:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829477.896951:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085839744:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829477.896955:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085839872:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829477.896972:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829477.896979:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829477/real 1787829477] req@ffff9e35c498f480 x1874672085839872/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829502 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829477.896987:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f480 x1874672085839872/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829502 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829477.896993:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829477.896995:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829477.896998:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085839872:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829478.024000:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829478.024000:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829478.024001:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829478.024060:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829479.047937:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829479.047946:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829479.047947:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:0.0:1787829479.048011:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829480.071901:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829480.071954:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829480.071957:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829480.072010:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829481.095916:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829481.095928:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829481.095951:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787829481.095953:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:3.0:1787829482.119897:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829482.119898:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829482.119933:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787829482.119994:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829483.015935:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829483.015946:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829483.015951:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829483.015963:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2918 00000100:00080000:2.0:1787829483.015974:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829483.015982:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787829483.016024:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e940 x1874672085840000/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829483.016054:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829483.016058:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829483.016061:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829483.016066:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2918 00000100:00080000:2.0:1787829483.016069:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829483.016074:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829483.016086:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829483.016099:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f0c0 x1874672085840128/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829483.016112:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829483.016116:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829483.016119:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787829483.016128:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085840000:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1787829483.016128:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2918 00000100:00080000:2.0:1787829483.016136:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829483.016146:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829483.016152:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829483.016164:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c000 x1874672085840256/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829483.016174:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829483.016178:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829483.016181:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829483.016187:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2918 00000100:00080000:2.0:1787829483.016191:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829483.016195:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829483.016200:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:2.0:1787829483.016216:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cb40 x1874672085840384/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787829483.016218:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1787829483.016230:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829483.016234:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829483.016237:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829483.016244:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2918 00000100:00080000:2.0:1787829483.016248:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829483.016252:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829483.016257:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:2.0:1787829483.016275:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498fc00 x1874672085840512/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829483.016276:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829483/real 1787829483] req@ffff9e35c498e940 x1874672085840000/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829508 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829483.016307:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829483.016318:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829483.016322:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829483.016329:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2918 00000100:00100000:1.0:1787829483.016334:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e940 x1874672085840000/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829508 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829483.016335:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829483.016343:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829483.016351:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00080000:1.0:1787829483.016365:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:2.0:1787829483.016368:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e580 x1874672085840640/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829483.016375:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:2.0:1787829483.016387:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829483.016397:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:1.0:1787829483.016400:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085840000:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1787829483.016400:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829483.016412:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2918 00000100:00080000:2.0:1787829483.016418:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829483.016424:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829483.016428:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829483.016439:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e940 x1874672085840768/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829483.016447:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085840128:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1787829483.016454:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829483.016458:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829483.016460:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829483.016466:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2918 00000100:00080000:2.0:1787829483.016470:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829483.016476:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829483.016481:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000800:00000100:1.0:1787829483.016490:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829483.016502:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f480 x1874672085840896/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829483.016514:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829483/real 1787829483] req@ffff9e35c498f0c0 x1874672085840128/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829508 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829483.016520:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829483.016531:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829483.016535:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829483.016545:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2918 00000100:00100000:1.0:1787829483.016551:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f0c0 x1874672085840128/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829508 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829483.016551:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829483.016562:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829483.016572:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00080000:1.0:1787829483.016578:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829483.016585:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829483.016590:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085840128:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829483.016596:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e1c0 x1874672085841024/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829483.016606:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085840256:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787829483.016645:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829483.016658:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829483/real 1787829483] req@ffff9e35c498c000 x1874672085840256/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829508 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829483.016670:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c000 x1874672085840256/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829508 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829483.016676:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829483.016678:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829483.016680:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085840256:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829483.016684:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085840384:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829483.016698:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829483.016705:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829483/real 1787829483] req@ffff9e35c498cb40 x1874672085840384/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829508 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829483.016711:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498cb40 x1874672085840384/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829508 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829483.016716:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829483.016718:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829483.016720:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085840384:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829483.016723:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085840512:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829483.016732:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829483.016736:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829483/real 1787829483] req@ffff9e35c498fc00 x1874672085840512/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829508 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829483.016742:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498fc00 x1874672085840512/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829508 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829483.016754:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829483.016757:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829483.016760:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085840512:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829483.016766:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085840640:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787829483.016784:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829483.016794:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829483/real 1787829483] req@ffff9e35c498e580 x1874672085840640/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829508 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829483.016837:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e580 x1874672085840640/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829508 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829483.016843:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829483.016845:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829483.016847:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085840640:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829483.016884:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085840768:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787829483.016904:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829483.016914:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829483/real 1787829483] req@ffff9e35c498e940 x1874672085840768/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829508 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829483.016925:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e940 x1874672085840768/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829508 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829483.016932:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829483.016935:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829483.016938:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085840768:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829483.016944:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085840896:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829483.016956:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829483.016963:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829483/real 1787829483] req@ffff9e35c498f480 x1874672085840896/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829508 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829483.016969:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f480 x1874672085840896/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829508 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829483.016974:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829483.016976:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829483.016978:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085840896:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829483.016981:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085841024:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829483.016989:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829483.016994:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829483/real 1787829483] req@ffff9e35c498e1c0 x1874672085841024/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829508 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829483.017000:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e1c0 x1874672085841024/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829508 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829483.017005:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829483.017008:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829483.017010:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085841024:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829483.143877:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829483.143883:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829483.143925:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829483.143929:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829484.167915:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787829484.167921:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829484.167934:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829484.168014:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829485.191914:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829485.191955:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829485.191955:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787829485.191968:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829486.215909:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829486.215909:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829486.215970:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829486.216010:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829487.239937:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829487.239976:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829487.239979:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829487.239999:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00080000:3.0:1787829488.135960:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829488.135973:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787829488.135978:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829488.135992:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2923 00000100:00080000:3.0:1787829488.136005:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829488.136014:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787829488.136060:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498000 x1874672085841152/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829488.136094:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829488.136099:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829488.136101:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829488.136106:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2923 00000100:00080000:3.0:1787829488.136110:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829488.136115:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829488.136127:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829488.136143:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649ad00 x1874672085841280/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829488.136145:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085841152:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1787829488.136163:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829488.136168:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787829488.136170:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829488.136175:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2923 00000100:00080000:3.0:1787829488.136179:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829488.136183:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829488.136189:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829488.136203:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c64983c0 x1874672085841408/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829488.136214:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829488.136218:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787829488.136220:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829488.136225:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2923 00000800:00000100:1.0:1787829488.136231:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1787829488.136232:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829488.136239:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829488.136245:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:3.0:1787829488.136260:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c64992c0 x1874672085841536/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829488.136271:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829488.136275:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787829488.136277:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829488.136283:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2923 00000100:00080000:3.0:1787829488.136288:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1787829488.136291:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829488/real 1787829488] req@ffff9e35c6498000 x1874672085841152/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829513 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829488.136296:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829488.136306:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:1.0:1787829488.136315:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498000 x1874672085841152/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829513 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829488.136325:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b480 x1874672085841664/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829488.136337:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829488.136341:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829488.136344:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:3.0:1787829488.136349:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:1.0:1787829488.136352:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085841152:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1787829488.136353:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829488.136362:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2923 00000100:00080000:3.0:1787829488.136366:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829488.136371:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829488.136376:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829488.136382:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085841280:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787829488.136391:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498780 x1874672085841792/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829488.136406:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1787829488.136408:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1787829488.136414:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787829488.136417:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787829488.136422:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829488/real 1787829488] req@ffff9e35c649ad00 x1874672085841280/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829513 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829488.136424:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2923 00000100:00080000:3.0:1787829488.136432:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1787829488.136440:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649ad00 x1874672085841280/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829513 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829488.136440:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829488.136450:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00080000:1.0:1787829488.136452:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829488.136458:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829488.136461:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085841280:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787829488.136468:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498f00 x1874672085841920/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829488.136472:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c64983c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085841408:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1787829488.136487:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829488.136492:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000100:1.0:1787829488.136493:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1787829488.136497:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829488.136504:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2923 00000100:00080000:3.0:1787829488.136509:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829488.136517:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787829488.136518:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829488/real 1787829488] req@ffff9e35c64983c0 x1874672085841408/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829513 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1787829488.136525:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:3.0:1787829488.136542:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499a40 x1874672085842048/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829488.136544:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c64983c0 x1874672085841408/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829513 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829488.136574:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829488.136575:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829488.136581:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:3.0:1787829488.136583:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:1.0:1787829488.136587:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c64983c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085841408:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1787829488.136589:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787829488.136598:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c64992c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085841536:192.168.203.141@tcp:8:kworker.0 00000100:00080000:3.0:1787829488.136600:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2923 00000100:00080000:3.0:1787829488.136606:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829488.136612:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829488.136618:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000800:00000100:1.0:1787829488.136623:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787829488.136635:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a1c0 x1874672085842176/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829488.136636:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829488/real 1787829488] req@ffff9e35c64992c0 x1874672085841536/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829513 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829488.136670:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c64992c0 x1874672085841536/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829513 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829488.136678:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829488.136680:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829488.136682:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c64992c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085841536:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829488.136686:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085841664:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829488.136699:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829488.136706:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829488/real 1787829488] req@ffff9e35c649b480 x1874672085841664/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829513 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829488.136714:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b480 x1874672085841664/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829513 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829488.136720:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829488.136722:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829488.136724:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085841664:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829488.136739:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085841792:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787829488.136750:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829488.136756:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829488/real 1787829488] req@ffff9e35c6498780 x1874672085841792/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829513 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829488.136764:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498780 x1874672085841792/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829513 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829488.136769:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829488.136771:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829488.136773:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085841792:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829488.136776:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085841920:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787829488.136786:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829488.136792:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829488/real 1787829488] req@ffff9e35c6498f00 x1874672085841920/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829513 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829488.136850:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498f00 x1874672085841920/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829513 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829488.136867:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829488.136870:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829488.136874:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085841920:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829488.136881:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085842048:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829488.136903:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829488.136914:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829488/real 1787829488] req@ffff9e35c6499a40 x1874672085842048/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829513 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829488.136927:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499a40 x1874672085842048/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829513 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829488.136936:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829488.136939:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829488.136942:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085842048:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829488.136949:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085842176:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829488.136970:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829488.136982:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829488/real 1787829488] req@ffff9e35c649a1c0 x1874672085842176/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829513 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829488.136994:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a1c0 x1874672085842176/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829513 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829488.137003:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829488.137006:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829488.137010:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085842176:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829488.263891:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829488.263899:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829488.263922:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787829488.264008:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829489.287920:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829489.287920:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829489.287962:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787829489.287990:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829490.311912:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829490.311944:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829490.311951:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787829490.312010:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829491.335943:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829491.335943:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829491.335975:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829491.335981:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829492.359894:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829492.359898:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829492.359910:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829492.359948:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829493.255941:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829493.255951:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829493.255955:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829493.255966:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2928 00000100:00080000:2.0:1787829493.255978:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829493.255985:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787829493.256025:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498d680 x1874672085842304/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829493.256049:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829493.256052:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829493.256054:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829493.256064:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2928 00000100:00080000:2.0:1787829493.256067:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829493.256071:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829493.256080:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829493.256089:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cf00 x1874672085842432/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829493.256103:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829493.256105:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829493.256108:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787829493.256109:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085842304:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1787829493.256113:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2928 00000100:00080000:2.0:1787829493.256118:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829493.256122:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829493.256125:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829493.256136:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f840 x1874672085842560/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829493.256142:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829493.256145:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829493.256146:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829493.256149:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2928 00000100:00080000:2.0:1787829493.256152:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829493.256155:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829493.256158:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:2.0:1787829493.256165:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498ed00 x1874672085842688/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829493.256171:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829493.256174:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829493.256175:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829493.256180:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2928 00000800:00000100:1.0:1787829493.256181:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1787829493.256186:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829493.256190:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829493.256194:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:2.0:1787829493.256202:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498de00 x1874672085842816/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829493.256208:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829493.256210:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829493.256212:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829493.256215:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2928 00000100:00080000:2.0:1787829493.256218:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829493.256220:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829493.256224:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829493.256228:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829493/real 1787829493] req@ffff9e35c498d680 x1874672085842304/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829518 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829493.256233:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c780 x1874672085842944/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829493.256252:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829493.256255:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:1.0:1787829493.256259:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498d680 x1874672085842304/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829518 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829493.256259:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829493.256263:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2928 00000100:00080000:2.0:1787829493.256268:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829493.256273:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829493.256279:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00080000:1.0:1787829493.256281:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829493.256286:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829493.256289:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c3c0 x1874672085843072/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829493.256295:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085842304:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1787829493.256311:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829493.256314:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829493.256315:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829493.256319:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2928 00000100:00080000:2.0:1787829493.256321:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829493.256324:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829493.256326:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:2.0:1787829493.256332:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498d680 x1874672085843200/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829493.256338:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787829493.256341:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085842432:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1787829493.256342:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829493.256345:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829493.256351:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2928 00000100:00080000:2.0:1787829493.256353:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829493.256355:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829493.256358:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:2.0:1787829493.256367:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e1c0 x1874672085843328/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787829493.256378:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829493.256391:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829493/real 1787829493] req@ffff9e35c498cf00 x1874672085842432/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829518 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829493.256401:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498cf00 x1874672085842432/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829518 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829493.256413:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829493.256416:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829493.256418:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085842432:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829493.256422:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085842560:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787829493.256435:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829493.256441:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829493/real 1787829493] req@ffff9e35c498f840 x1874672085842560/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829518 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829493.256449:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f840 x1874672085842560/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829518 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829493.256455:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829493.256456:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829493.256458:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085842560:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829493.256462:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085842688:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829493.256473:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829493.256480:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829493/real 1787829493] req@ffff9e35c498ed00 x1874672085842688/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829518 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829493.256487:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498ed00 x1874672085842688/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829518 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829493.256493:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829493.256494:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829493.256497:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085842688:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829493.256501:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085842816:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829493.256511:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829493.256517:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829493/real 1787829493] req@ffff9e35c498de00 x1874672085842816/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829518 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829493.256524:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498de00 x1874672085842816/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829518 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829493.256530:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829493.256531:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829493.256533:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085842816:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829493.256537:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085842944:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787829493.256547:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829493.256552:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829493/real 1787829493] req@ffff9e35c498c780 x1874672085842944/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829518 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829493.256560:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c780 x1874672085842944/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829518 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829493.256565:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829493.256567:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829493.256569:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085842944:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829493.256572:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085843072:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787829493.256581:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829493.256587:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829493/real 1787829493] req@ffff9e35c498c3c0 x1874672085843072/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829518 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829493.256594:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c3c0 x1874672085843072/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829518 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829493.256599:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829493.256601:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829493.256603:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085843072:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829493.256620:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085843200:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829493.256630:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829493.256637:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829493/real 1787829493] req@ffff9e35c498d680 x1874672085843200/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829518 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829493.256645:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498d680 x1874672085843200/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829518 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829493.256650:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829493.256652:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829493.256654:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085843200:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829493.256657:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085843328:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829493.256667:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829493.256672:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829493/real 1787829493] req@ffff9e35c498e1c0 x1874672085843328/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829518 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829493.256679:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e1c0 x1874672085843328/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829518 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829493.256685:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829493.256687:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829493.256689:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085843328:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829493.383879:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829493.383879:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829493.383891:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787829493.383932:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829494.407903:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829494.407905:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829494.407916:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829494.407951:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829495.431903:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829495.431926:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829495.431959:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787829495.431978:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829496.455910:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829496.455930:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829496.455947:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787829496.455974:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829497.479942:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829497.479943:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829497.479972:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829497.480090:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00080000:0.0:1787829498.375927:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829498.375934:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787829498.375938:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829498.375947:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2933 00000100:00080000:0.0:1787829498.375959:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829498.375968:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787829498.376002:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718780 x1874672085843456/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829498.376026:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829498.376029:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829498.376031:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829498.376034:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2933 00000100:00080000:0.0:1787829498.376037:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829498.376042:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829498.376053:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829498.376066:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085843456:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1787829498.376068:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718f00 x1874672085843584/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829498.376078:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829498.376080:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787829498.376081:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829498.376084:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2933 00000100:00080000:0.0:1787829498.376087:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829498.376089:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829498.376093:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829498.376109:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a940 x1874672085843712/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829498.376116:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829498.376119:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829498.376121:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829498.376125:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2933 00000100:00080000:0.0:1787829498.376127:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829498.376129:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829498.376132:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:0.0:1787829498.376141:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7183c0 x1874672085843840/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787829498.376142:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787829498.376152:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829498.376155:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829498.376157:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829498.376162:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2933 00000100:00080000:0.0:1787829498.376168:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829498.376174:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829498.376181:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:0.0:1787829498.376194:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da719a40 x1874672085843968/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829498.376206:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829498.376210:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829498.376213:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787829498.376215:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829498/real 1787829498] req@ffff9e35da718780 x1874672085843456/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829523 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829498.376220:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2933 00000100:00080000:0.0:1787829498.376227:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829498.376234:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787829498.376239:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718780 x1874672085843456/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829523 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1787829498.376240:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00080000:1.0:1787829498.376252:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829498.376257:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829498.376258:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a580 x1874672085844096/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829498.376261:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085843456:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1787829498.376272:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829498.376274:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787829498.376276:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829498.376280:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2933 00000100:00100000:1.0:1787829498.376281:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085843584:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787829498.376283:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829498.376289:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829498.376292:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829498.376301:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718000 x1874672085844224/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787829498.376306:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787829498.376316:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829498.376319:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829498.376320:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829498.376325:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2933 00000100:00100000:1.0:1787829498.376325:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829498/real 1787829498] req@ffff9e35da718f00 x1874672085843584/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829523 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829498.376329:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829498.376335:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829498.376343:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:1.0:1787829498.376353:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672085843584/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829523 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829498.376360:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718b40 x1874672085844352/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829498.376371:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829498.376373:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829498.376375:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829498.376378:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829498.376380:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:0.0:1787829498.376385:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2933 00000100:00100000:1.0:1787829498.376386:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085843584:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787829498.376389:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829498.376393:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787829498.376395:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085843712:192.168.203.141@tcp:38:kworker.0 00000008:00100000:0.0:1787829498.376400:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:0.0:1787829498.376410:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a1c0 x1874672085844480/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787829498.376423:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829498.376439:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829498/real 1787829498] req@ffff9e35da71a940 x1874672085843712/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829523 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829498.376450:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672085843712/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829523 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829498.376455:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829498.376457:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829498.376459:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085843712:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829498.376463:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085843840:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829498.376477:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829498.376483:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829498/real 1787829498] req@ffff9e35da7183c0 x1874672085843840/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829523 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829498.376489:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672085843840/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829523 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829498.376498:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829498.376500:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829498.376503:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085843840:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829498.376506:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085843968:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829498.376515:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829498.376520:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829498/real 1787829498] req@ffff9e35da719a40 x1874672085843968/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829523 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829498.376526:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672085843968/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829523 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829498.376531:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829498.376532:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829498.376534:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085843968:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829498.376537:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085844096:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787829498.376552:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829498.376559:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829498/real 1787829498] req@ffff9e35da71a580 x1874672085844096/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829523 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829498.376566:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672085844096/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829523 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829498.376571:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829498.376573:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829498.376575:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085844096:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829498.376598:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085844224:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787829498.376622:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829498.376637:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829498/real 1787829498] req@ffff9e35da718000 x1874672085844224/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829523 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829498.376652:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718000 x1874672085844224/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829523 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829498.376664:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829498.376668:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829498.376671:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085844224:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829498.376679:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085844352:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829498.376701:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829498.376708:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829498/real 1787829498] req@ffff9e35da718b40 x1874672085844352/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829523 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829498.376715:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718b40 x1874672085844352/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829523 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829498.376720:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829498.376722:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829498.376723:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085844352:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829498.376726:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085844480:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829498.376734:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829498.376739:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829498/real 1787829498] req@ffff9e35da71a1c0 x1874672085844480/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829523 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829498.376745:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a1c0 x1874672085844480/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829523 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829498.376750:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829498.376752:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829498.376763:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085844480:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829498.503901:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829498.503916:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829498.503953:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787829498.504010:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829499.527911:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829499.527911:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787829499.527945:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829499.527986:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829500.551890:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829500.551890:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829500.551935:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787829500.551946:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829501.575881:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829501.575881:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829501.575883:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787829501.575950:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829502.599891:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829502.599891:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829502.599892:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787829502.599963:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829503.495921:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829503.495932:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829503.495937:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829503.495947:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2938 00000100:00080000:1.0:1787829503.495958:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829503.495964:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787829503.496004:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5a40 x1874672085844608/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829503.496030:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829503.496034:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829503.496035:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829503.496038:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2938 00000100:00080000:1.0:1787829503.496041:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829503.496044:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829503.496054:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829503.496064:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5e00 x1874672085844736/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829503.496073:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829503.496075:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829503.496077:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829503.496080:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2938 00000100:00080000:1.0:1787829503.496084:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787829503.496086:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085844608:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787829503.496089:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829503.496097:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829503.496106:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4b40 x1874672085844864/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829503.496112:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829503.496115:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829503.496117:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829503.496119:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2938 00000100:00080000:1.0:1787829503.496122:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829503.496125:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829503.496128:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:1.0:1787829503.496136:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f70c0 x1874672085844992/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829503.496147:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829503.496150:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829503.496152:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829503.496154:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2938 00000100:00080000:1.0:1787829503.496156:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829503.496158:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829503.496162:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000800:00000100:2.0:1787829503.496166:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829503.496174:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f43c0 x1874672085845120/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829503.496182:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829503.496185:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829503.496186:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829503.496190:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2938 00000100:00080000:1.0:1787829503.496192:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829503.496194:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829503.496197:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829503.496208:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7840 x1874672085845248/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829503.496214:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829503/real 1787829503] req@ffff9e35da6f5a40 x1874672085844608/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829528 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829503.496218:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829503.496225:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829503.496229:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829503.496234:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2938 00000100:00100000:2.0:1787829503.496241:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5a40 x1874672085844608/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829528 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829503.496246:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829503.496251:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829503.496252:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:2.0:1787829503.496258:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000008:00100000:1.0:1787829503.496259:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829503.496265:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085844608:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1787829503.496272:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f61c0 x1874672085845376/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829503.496280:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829503.496283:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829503.496284:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829503.496286:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2938 00000100:00080000:1.0:1787829503.496289:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787829503.496293:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085844736:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787829503.496295:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829503.496303:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:1.0:1787829503.496314:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4f00 x1874672085845504/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829503.496323:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1787829503.496323:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787829503.496330:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829503.496334:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787829503.496338:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829503/real 1787829503] req@ffff9e35da6f5e00 x1874672085844736/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829528 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829503.496340:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2938 00000100:00100000:2.0:1787829503.496352:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5e00 x1874672085844736/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829528 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829503.496356:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829503.496361:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829503.496366:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:2.0:1787829503.496367:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829503.496374:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085844736:192.168.203.141@tcp:38:kworker.0 00000008:00100000:1.0:1787829503.496378:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:2.0:1787829503.496381:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085844864:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829503.496397:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6580 x1874672085845632/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829503.496401:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829503.496411:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829503/real 1787829503] req@ffff9e35da6f4b40 x1874672085844864/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829528 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829503.496442:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4b40 x1874672085844864/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829528 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829503.496450:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829503.496458:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829503.496461:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085844864:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829503.496467:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085844992:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829503.496480:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829503.496487:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829503/real 1787829503] req@ffff9e35da6f70c0 x1874672085844992/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829528 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829503.496495:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f70c0 x1874672085844992/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829528 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829503.496500:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829503.496502:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829503.496504:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085844992:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829503.496508:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085845120:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829503.496524:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829503.496531:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829503/real 1787829503] req@ffff9e35da6f43c0 x1874672085845120/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829528 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829503.496539:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f43c0 x1874672085845120/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829528 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829503.496545:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829503.496547:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829503.496549:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085845120:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829503.496552:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085845248:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787829503.496562:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829503.496568:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829503/real 1787829503] req@ffff9e35da6f7840 x1874672085845248/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829528 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829503.496576:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7840 x1874672085845248/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829528 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829503.496581:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829503.496583:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829503.496584:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085845248:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829503.496588:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085845376:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787829503.496597:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829503.496603:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829503/real 1787829503] req@ffff9e35da6f61c0 x1874672085845376/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829528 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829503.496610:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f61c0 x1874672085845376/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829528 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829503.496616:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829503.496618:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829503.496620:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085845376:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829503.496636:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085845504:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829503.496647:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829503.496653:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829503/real 1787829503] req@ffff9e35da6f4f00 x1874672085845504/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829528 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829503.496660:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4f00 x1874672085845504/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829528 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829503.496666:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829503.496667:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829503.496669:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085845504:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829503.496673:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085845632:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829503.496682:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829503.496688:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829503/real 1787829503] req@ffff9e35da6f6580 x1874672085845632/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829528 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829503.496695:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6580 x1874672085845632/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829528 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829503.496700:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829503.496702:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829503.496705:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085845632:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829503.623919:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829503.623919:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829503.623920:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787829503.624009:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829504.647932:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829504.647932:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829504.647932:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787829504.648010:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829505.671877:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829505.671877:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829505.671894:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787829505.671973:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829506.695930:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829506.695930:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829506.695930:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787829506.695992:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829507.719895:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829507.719895:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829507.719913:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787829507.719966:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829508.615895:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829508.615905:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787829508.615909:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829508.615920:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2943 00000100:00080000:0.0:1787829508.615931:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829508.615937:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787829508.615977:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7192c0 x1874672085845760/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829508.616005:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829508.616008:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829508.616010:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829508.616013:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2943 00000100:00080000:0.0:1787829508.616015:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829508.616018:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829508.616028:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829508.616039:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a1c0 x1874672085845888/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829508.616044:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085845760:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1787829508.616053:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829508.616056:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787829508.616058:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829508.616061:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2943 00000100:00080000:0.0:1787829508.616063:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829508.616065:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829508.616068:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829508.616076:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718b40 x1874672085846016/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829508.616082:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829508.616085:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829508.616086:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829508.616089:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2943 00000100:00080000:0.0:1787829508.616093:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829508.616096:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829508.616100:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:0.0:1787829508.616109:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718000 x1874672085846144/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829508.616116:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787829508.616122:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829508.616125:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829508.616127:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829508.616131:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2943 00000100:00080000:0.0:1787829508.616136:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829508.616139:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829508.616142:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:0.0:1787829508.616151:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a580 x1874672085846272/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829508.616157:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787829508.616160:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829508/real 1787829508] req@ffff9e35da7192c0 x1874672085845760/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829533 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829508.616161:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829508.616165:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829508.616173:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2943 00000100:00080000:0.0:1787829508.616178:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829508.616181:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829508.616185:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829508.616188:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7192c0 x1874672085845760/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829533 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829508.616196:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da719a40 x1874672085846400/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829508.616201:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829508.616209:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829508.616216:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085845760:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1787829508.616219:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829508.616224:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787829508.616225:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829508.616229:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2943 00000100:00080000:0.0:1787829508.616231:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829508.616233:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829508.616236:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829508.616243:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7183c0 x1874672085846528/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829508.616247:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085845888:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787829508.616260:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829508.616263:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829508.616264:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829508.616268:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2943 00000100:00080000:0.0:1787829508.616273:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1787829508.616276:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787829508.616279:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829508.616284:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:2.0:1787829508.616292:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829508/real 1787829508] req@ffff9e35da71a1c0 x1874672085845888/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829533 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829508.616297:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a940 x1874672085846656/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829508.616315:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787829508.616318:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a1c0 x1874672085845888/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829533 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829508.616319:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829508.616322:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829508.616330:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2943 00000100:00080000:2.0:1787829508.616333:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829508.616334:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829508.616341:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:0.0:1787829508.616342:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829508.616348:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:2.0:1787829508.616348:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085845888:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829508.616359:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085846016:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829508.616362:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718f00 x1874672085846784/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829508.616391:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829508.616405:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829508/real 1787829508] req@ffff9e35da718b40 x1874672085846016/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829533 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829508.616419:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718b40 x1874672085846016/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829533 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829508.616428:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829508.616430:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829508.616433:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085846016:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829508.616437:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085846144:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829508.616455:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829508.616463:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829508/real 1787829508] req@ffff9e35da718000 x1874672085846144/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829533 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829508.616470:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718000 x1874672085846144/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829533 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829508.616477:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829508.616479:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829508.616481:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085846144:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829508.616485:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085846272:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829508.616496:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829508.616502:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829508/real 1787829508] req@ffff9e35da71a580 x1874672085846272/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829533 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829508.616509:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672085846272/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829533 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829508.616515:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829508.616517:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829508.616519:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085846272:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829508.616522:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085846400:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787829508.616533:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829508.616539:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829508/real 1787829508] req@ffff9e35da719a40 x1874672085846400/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829533 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829508.616546:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672085846400/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829533 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829508.616552:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829508.616553:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829508.616555:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085846400:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829508.616570:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085846528:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787829508.616581:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829508.616587:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829508/real 1787829508] req@ffff9e35da7183c0 x1874672085846528/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829533 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829508.616600:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672085846528/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829533 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829508.616605:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829508.616607:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829508.616609:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085846528:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829508.616612:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085846656:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829508.616622:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829508.616628:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829508/real 1787829508] req@ffff9e35da71a940 x1874672085846656/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829533 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829508.616636:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672085846656/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829533 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829508.616641:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829508.616643:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829508.616645:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085846656:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829508.616648:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085846784:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829508.616658:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829508.616663:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829508/real 1787829508] req@ffff9e35da718f00 x1874672085846784/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829533 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829508.616671:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672085846784/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829533 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829508.616681:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829508.616695:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829508.616698:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085846784:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829508.743942:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829508.743942:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829508.743985:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:0.0:1787829508.743995:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829509.767891:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829509.767901:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829509.767925:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787829509.767982:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829510.791916:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829510.791916:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787829510.791960:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829510.792033:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829511.815885:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:3.0:1787829511.815924:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829511.815925:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:0.0:1787829511.816022:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829512.839966:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829512.839982:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829512.840015:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787829512.840033:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829513.735951:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829513.735961:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829513.735965:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829513.735973:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2949 00000100:00080000:2.0:1787829513.735982:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829513.735987:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787829513.736018:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f480 x1874672085846912/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829513.736044:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829513.736048:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829513.736049:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829513.736053:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2949 00000100:00080000:2.0:1787829513.736057:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829513.736061:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829513.736072:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829513.736087:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e940 x1874672085847040/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829513.736098:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085846912:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1787829513.736102:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829513.736111:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829513.736113:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829513.736116:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2949 00000100:00080000:2.0:1787829513.736119:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829513.736121:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829513.736125:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829513.736142:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e580 x1874672085847168/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829513.736148:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829513.736151:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829513.736152:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829513.736157:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2949 00000100:00080000:2.0:1787829513.736158:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829513.736160:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829513.736164:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:2.0:1787829513.736171:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498fc00 x1874672085847296/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829513.736177:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829513.736180:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829513.736181:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1787829513.736190:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1787829513.736194:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2949 00000100:00080000:2.0:1787829513.736198:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829513.736200:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829513.736203:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:2.0:1787829513.736210:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cb40 x1874672085847424/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829513.736217:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829513.736221:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829513.736222:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829513.736224:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2949 00000100:00080000:2.0:1787829513.736226:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829513.736229:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829513.736231:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829513.736238:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c000 x1874672085847552/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829513.736245:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829513.736247:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829513.736250:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787829513.736253:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829513/real 1787829513] req@ffff9e35c498f480 x1874672085846912/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829538 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829513.736254:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2949 00000100:00080000:2.0:1787829513.736263:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829513.736274:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829513.736288:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829513.736299:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f0c0 x1874672085847680/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829513.736304:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f480 x1874672085846912/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829538 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829513.736313:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829513.736325:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829513.736331:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829513.736340:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2949 00000100:00080000:2.0:1787829513.736343:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829513.736346:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:1.0:1787829513.736348:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:2.0:1787829513.736350:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00080000:1.0:1787829513.736359:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829513.736364:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e1c0 x1874672085847808/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829513.736371:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085846912:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1787829513.736376:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829513.736385:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829513.736389:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829513.736393:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2949 00000100:00080000:2.0:1787829513.736394:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829513.736397:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829513.736400:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:2.0:1787829513.736408:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498d680 x1874672085847936/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829513.736424:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085847040:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787829513.736454:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829513.736466:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829513/real 1787829513] req@ffff9e35c498e940 x1874672085847040/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829538 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829513.736481:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e940 x1874672085847040/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829538 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829513.736488:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829513.736490:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829513.736493:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085847040:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829513.736498:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085847168:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787829513.736510:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829513.736517:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829513/real 1787829513] req@ffff9e35c498e580 x1874672085847168/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829538 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829513.736524:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e580 x1874672085847168/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829538 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829513.736530:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829513.736532:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829513.736534:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085847168:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829513.736537:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085847296:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829513.736549:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829513.736555:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829513/real 1787829513] req@ffff9e35c498fc00 x1874672085847296/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829538 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829513.736562:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498fc00 x1874672085847296/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829538 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829513.736572:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829513.736574:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829513.736576:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085847296:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829513.736580:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085847424:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829513.736590:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829513.736596:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829513/real 1787829513] req@ffff9e35c498cb40 x1874672085847424/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829538 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829513.736603:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498cb40 x1874672085847424/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829538 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829513.736609:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829513.736611:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829513.736613:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085847424:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829513.736617:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085847552:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787829513.736627:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829513.736633:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829513/real 1787829513] req@ffff9e35c498c000 x1874672085847552/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829538 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829513.736640:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c000 x1874672085847552/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829538 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829513.736646:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829513.736663:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829513.736666:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085847552:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829513.736671:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085847680:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787829513.736687:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829513.736695:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829513/real 1787829513] req@ffff9e35c498f0c0 x1874672085847680/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829538 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829513.736703:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f0c0 x1874672085847680/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829538 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829513.736708:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829513.736710:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829513.736712:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085847680:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829513.736716:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085847808:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829513.736726:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829513.736732:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829513/real 1787829513] req@ffff9e35c498e1c0 x1874672085847808/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829538 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829513.736739:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e1c0 x1874672085847808/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829538 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829513.736745:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829513.736749:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829513.736752:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085847808:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829513.736759:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085847936:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829513.736778:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829513.736790:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829513/real 1787829513] req@ffff9e35c498d680 x1874672085847936/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829538 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829513.736826:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498d680 x1874672085847936/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829538 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829513.736832:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829513.736835:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829513.736837:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085847936:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829513.863919:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829513.863929:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829513.863932:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:0.0:1787829513.863964:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829514.887924:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829514.887950:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829514.887954:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787829514.887967:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829515.911934:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829515.911942:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829515.911950:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:1.0:1787829515.912018:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829516.935947:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829516.935948:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829516.935977:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787829516.936034:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829517.959924:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829517.959924:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:0.0:1787829517.959985:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829517.959995:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787829518.855966:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829518.855979:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787829518.855984:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829518.855997:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2954 00000100:00080000:3.0:1787829518.856010:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829518.856018:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787829518.856064:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a580 x1874672085848064/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829518.856097:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829518.856102:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829518.856104:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829518.856109:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2954 00000100:00080000:3.0:1787829518.856113:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829518.856119:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829518.856131:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829518.856145:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085848064:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1787829518.856149:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b0c0 x1874672085848192/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829518.856177:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829518.856181:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787829518.856183:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829518.856189:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2954 00000100:00080000:3.0:1787829518.856194:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829518.856198:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829518.856203:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829518.856218:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499e00 x1874672085848320/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829518.856228:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829518.856233:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787829518.856235:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829518.856239:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2954 00000100:00080000:3.0:1787829518.856243:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829518.856248:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829518.856255:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:1.0:1787829518.856266:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085848192:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787829518.856273:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a940 x1874672085848448/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829518.856287:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829518.856291:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787829518.856295:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787829518.856298:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085848320:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1787829518.856304:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2954 00000100:00080000:3.0:1787829518.856313:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829518.856318:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829518.856325:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:1.0:1787829518.856327:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085848448:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787829518.856347:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499680 x1874672085848576/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829518.856366:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829518.856371:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829518.856374:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787829518.856381:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085848576:192.168.203.141@tcp:8:kworker.0 00000100:00080000:3.0:1787829518.856383:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2954 00000100:00080000:3.0:1787829518.856391:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829518.856396:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829518.856403:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829518.856421:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b840 x1874672085848704/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829518.856446:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829518.856451:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787829518.856453:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829518.856458:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2954 00000100:00080000:3.0:1787829518.856463:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829518.856468:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829518.856474:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829518.856485:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085848704:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787829518.856491:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649bc00 x1874672085848832/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829518.856505:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829518.856509:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787829518.856511:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829518.856517:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2954 00000100:00080000:3.0:1787829518.856522:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1787829518.856525:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085848832:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1787829518.856531:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829518.856539:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:3.0:1787829518.856556:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498b40 x1874672085848960/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829518.856571:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829518.856575:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787829518.856577:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829518.856583:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2954 00000100:00080000:3.0:1787829518.856587:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1787829518.856592:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085848960:192.168.203.141@tcp:8:kworker.0 00000100:00080000:3.0:1787829518.856595:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829518.856605:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:3.0:1787829518.856637:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a1c0 x1874672085849088/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829518.856654:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085849088:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829518.983918:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829518.983936:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829518.983955:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829518.983976:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829520.007895:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829520.007926:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829520.007933:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787829520.008012:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829521.031899:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787829521.031955:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829521.031964:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829521.031968:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000400:00000100:2.0:1787829521.928004:0:2396:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.141:988 00000400:00000100:2.0:1787829521.928040:0:2396:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.141@tcp at host 192.168.203.141 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1787829521.928053:0:2396:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:2.0:1787829521.928099:0:2396:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000800:00000100:2.0:1787829521.928113:0:2396:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:1.0:1787829521.928124:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829518/real 1787829521] req@ffff9e35c649a1c0 x1874672085849088/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829543 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:2.0:1787829521.928124:0:2396:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:2.0:1787829521.928137:0:2396:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:1.0:1787829521.928148:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a1c0 x1874672085849088/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829543 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:2.0:1787829521.928153:0:2396:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:2.0:1787829521.928165:0:2396:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00080000:1.0:1787829521.928175:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000800:00000100:2.0:1787829521.928175:0:2396:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00080000:1.0:1787829521.928183:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000800:00000100:2.0:1787829521.928188:0:2396:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:1.0:1787829521.928193:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085849088:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829521.928203:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829518/real 1787829521] req@ffff9e35c6498b40 x1874672085848960/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829543 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829521.928221:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498b40 x1874672085848960/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829543 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829521.928230:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829521.928233:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829521.928237:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085848960:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829521.928243:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829518/real 1787829521] req@ffff9e35c649bc00 x1874672085848832/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829543 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829521.928255:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649bc00 x1874672085848832/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829543 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829521.928263:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829521.928267:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829521.928271:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085848832:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829521.928277:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829518/real 1787829521] req@ffff9e35c649b840 x1874672085848704/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829543 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829521.928287:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b840 x1874672085848704/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829543 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829521.928295:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829521.928299:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829521.928302:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085848704:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829521.928307:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829518/real 1787829521] req@ffff9e35c6499680 x1874672085848576/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829543 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829521.928317:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499680 x1874672085848576/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829543 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829521.928325:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829521.928327:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829521.928330:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085848576:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829521.928336:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829518/real 1787829521] req@ffff9e35c649a940 x1874672085848448/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829543 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829521.928347:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a940 x1874672085848448/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829543 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829521.928354:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829521.928357:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829521.928360:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085848448:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829521.928367:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829518/real 1787829521] req@ffff9e35c649b0c0 x1874672085848192/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829543 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829521.928384:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b0c0 x1874672085848192/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829543 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829521.928393:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829521.928396:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829521.928399:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085848192:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829521.928404:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829518/real 1787829521] req@ffff9e35c6499e00 x1874672085848320/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829543 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829521.928414:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499e00 x1874672085848320/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829543 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829521.928421:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829521.928424:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829521.928426:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085848320:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829521.928432:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829518/real 1787829521] req@ffff9e35c649a580 x1874672085848064/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829543 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829521.928441:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a580 x1874672085848064/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829543 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829521.928448:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829521.928450:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829521.928453:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085848064:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1787829522.055870:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829522.055873:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829522.055893:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829522.055922:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829523.079886:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829523.079886:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:0.0:1787829523.079913:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829523.079988:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00080000:3.0:1787829523.975929:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829523.975942:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787829523.975947:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829523.975960:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2959 00000100:00080000:3.0:1787829523.975981:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829523.975991:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787829523.976034:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499a40 x1874672085849216/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829523.976068:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829523.976072:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829523.976074:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829523.976079:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2959 00000100:00080000:3.0:1787829523.976082:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829523.976098:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829523.976111:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829523.976126:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498f00 x1874672085849344/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829523.976138:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829523.976142:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787829523.976147:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787829523.976148:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085849216:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1787829523.976166:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2959 00000100:00080000:3.0:1787829523.976176:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829523.976181:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829523.976188:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829523.976201:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498780 x1874672085849472/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829523.976212:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829523.976216:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787829523.976219:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829523.976223:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2959 00000100:00080000:3.0:1787829523.976228:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829523.976233:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829523.976238:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000800:00000100:1.0:1787829523.976244:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787829523.976262:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b480 x1874672085849600/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829523.976275:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829523.976280:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787829523.976282:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829523.976287:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2959 00000100:00080000:3.0:1787829523.976291:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829523.976296:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829523.976304:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:1.0:1787829523.976310:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829523/real 1787829523] req@ffff9e35c6499a40 x1874672085849216/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829548 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829523.976322:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c64992c0 x1874672085849728/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829523.976336:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787829523.976338:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499a40 x1874672085849216/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829548 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829523.976344:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829523.976349:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829523.976360:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2959 00000100:00080000:1.0:1787829523.976361:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829523.976366:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829523.976372:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829523.976381:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00080000:1.0:1787829523.976382:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829523.976390:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085849216:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1787829523.976401:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c64983c0 x1874672085849856/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829523.976414:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829523.976418:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787829523.976420:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787829523.976427:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085849344:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1787829523.976427:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2959 00000100:00080000:3.0:1787829523.976435:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829523.976442:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829523.976449:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1787829523.976467:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787829523.976468:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649ad00 x1874672085849984/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829523.976483:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787829523.976487:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829523/real 1787829523] req@ffff9e35c6498f00 x1874672085849344/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829548 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829523.976489:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787829523.976494:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829523.976503:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2959 00000100:00080000:3.0:1787829523.976510:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1787829523.976514:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498f00 x1874672085849344/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829548 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829523.976519:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829523.976529:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00080000:1.0:1787829523.976532:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829523.976539:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829523.976545:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085849344:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787829523.976547:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498000 x1874672085850112/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829523.976558:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085849472:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1787829523.976566:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829523.976572:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787829523.976574:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829523.976596:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2959 00000100:00080000:3.0:1787829523.976602:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1787829523.976604:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1787829523.976610:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829523.976617:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:1.0:1787829523.976623:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829523/real 1787829523] req@ffff9e35c6498780 x1874672085849472/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829548 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829523.976640:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498780 x1874672085849472/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829548 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829523.976650:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829523.976653:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829523.976657:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085849472:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829523.976664:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085849600:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829523.976685:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829523.976697:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829523/real 1787829523] req@ffff9e35c649b480 x1874672085849600/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829548 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829523.976710:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b480 x1874672085849600/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829548 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829523.976720:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829523.976723:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829523.976727:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085849600:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829523.976733:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c64992c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085849728:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829523.976752:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829523.976764:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829523/real 1787829523] req@ffff9e35c64992c0 x1874672085849728/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829548 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829523.976775:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c64992c0 x1874672085849728/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829548 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829523.976784:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829523.976787:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829523.976790:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c64992c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085849728:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829523.976796:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c64983c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085849856:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787829523.976853:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829523.976864:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829523/real 1787829523] req@ffff9e35c64983c0 x1874672085849856/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829548 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829523.976875:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c64983c0 x1874672085849856/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829548 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829523.976883:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829523.976886:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829523.976889:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c64983c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085849856:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829523.976910:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085849984:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787829523.976927:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829523.976937:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829523/real 1787829523] req@ffff9e35c649ad00 x1874672085849984/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829548 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829523.976948:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649ad00 x1874672085849984/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829548 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829523.976957:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829523.976961:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829523.976964:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085849984:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829523.976971:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085850112:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829523.976989:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829523.977000:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829523/real 1787829523] req@ffff9e35c6498000 x1874672085850112/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829548 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829523.977013:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498000 x1874672085850112/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829548 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829523.977022:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829523.977026:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829523.977029:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085850112:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787829523.977124:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499a40 x1874672085850240/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829523.977664:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085850240:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829523.977699:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829523.977715:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829523/real 1787829523] req@ffff9e35c6499a40 x1874672085850240/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829548 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829523.977730:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499a40 x1874672085850240/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829548 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829523.977739:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829523.977743:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829523.977747:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085850240:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829524.103951:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829524.103952:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829524.103996:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:0.0:1787829524.103999:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829525.127887:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829525.127887:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829525.127912:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829525.127948:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829526.151899:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829526.151899:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829526.151906:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:0.0:1787829526.151938:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829527.175891:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829527.175924:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829527.175944:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:0.0:1787829527.175992:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829528.199896:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829528.199929:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829528.199935:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829528.199974:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829529.095913:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829529.095923:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829529.095927:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829529.095937:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2964 00000100:00080000:2.0:1787829529.095947:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829529.095951:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787829529.095984:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c3c0 x1874672085850368/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829529.096013:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829529.096017:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829529.096019:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829529.096023:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2964 00000100:00080000:2.0:1787829529.096026:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829529.096030:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829529.096040:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829529.096055:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085850368:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1787829529.096055:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c780 x1874672085850496/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829529.096071:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829529.096074:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829529.096076:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829529.096080:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2964 00000100:00080000:2.0:1787829529.096084:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829529.096088:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829529.096093:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829529.096104:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498de00 x1874672085850624/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829529.096113:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829529.096116:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000100:1.0:1787829529.096118:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1787829529.096118:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829529.096124:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2964 00000100:00080000:2.0:1787829529.096127:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829529.096131:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829529.096135:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:2.0:1787829529.096145:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498ed00 x1874672085850752/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829529.096151:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829529.096153:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829529.096155:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787829529.096156:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829529/real 1787829529] req@ffff9e35c498c3c0 x1874672085850368/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829554 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829529.096159:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2964 00000100:00080000:2.0:1787829529.096164:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1787829529.096175:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c3c0 x1874672085850368/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829554 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829529.096176:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829529.096182:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00080000:1.0:1787829529.096190:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829529.096195:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829529.096195:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f840 x1874672085850880/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829529.096201:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085850368:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1787829529.096212:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829529.096214:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829529.096216:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829529.096219:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2964 00000100:00080000:2.0:1787829529.096220:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829529.096222:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829529.096225:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829529.096228:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085850496:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829529.096234:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cf00 x1874672085851008/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787829529.096246:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1787829529.096246:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829529.096251:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829529.096253:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829529.096257:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2964 00000100:00100000:1.0:1787829529.096260:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829529/real 1787829529] req@ffff9e35c498c780 x1874672085850496/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829554 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829529.096261:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829529.096269:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787829529.096274:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c780 x1874672085850496/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829554 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1787829529.096275:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00080000:1.0:1787829529.096286:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829529.096288:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829529.096294:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085850496:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829529.096294:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c3c0 x1874672085851136/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829529.096300:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085850624:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1787829529.096314:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1787829529.096319:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1787829529.096321:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829529.096323:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829529.096327:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2964 00000100:00080000:2.0:1787829529.096330:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1787829529.096331:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829529/real 1787829529] req@ffff9e35c498de00 x1874672085850624/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829554 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829529.096335:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829529.096341:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:1.0:1787829529.096352:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498de00 x1874672085850624/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829554 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829529.096354:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498d680 x1874672085851264/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829529.096384:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829529.096384:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829529.096389:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829529.096392:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:2.0:1787829529.096393:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787829529.096397:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085850624:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1787829529.096399:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2964 00000100:00080000:2.0:1787829529.096404:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1787829529.096407:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085850752:192.168.203.141@tcp:8:kworker.0 00000100:00080000:2.0:1787829529.096409:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829529.096414:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:2.0:1787829529.096422:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e1c0 x1874672085851392/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787829529.096430:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829529.096438:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829529/real 1787829529] req@ffff9e35c498ed00 x1874672085850752/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829554 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829529.096446:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498ed00 x1874672085850752/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829554 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829529.096451:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829529.096453:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829529.096454:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085850752:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829529.096458:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085850880:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829529.096466:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829529.096472:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829529/real 1787829529] req@ffff9e35c498f840 x1874672085850880/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829554 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829529.096478:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f840 x1874672085850880/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829554 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829529.096483:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829529.096484:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829529.096486:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085850880:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829529.096500:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085851008:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787829529.096509:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829529.096514:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829529/real 1787829529] req@ffff9e35c498cf00 x1874672085851008/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829554 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829529.096521:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498cf00 x1874672085851008/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829554 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829529.096526:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829529.096527:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829529.096529:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085851008:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829529.096531:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085851136:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787829529.096549:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829529.096559:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829529/real 1787829529] req@ffff9e35c498c3c0 x1874672085851136/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829554 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829529.096568:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c3c0 x1874672085851136/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829554 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829529.096573:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829529.096575:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829529.096577:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085851136:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829529.096580:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085851264:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829529.096592:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829529.096597:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829529/real 1787829529] req@ffff9e35c498d680 x1874672085851264/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829554 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829529.096603:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498d680 x1874672085851264/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829554 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829529.096608:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829529.096609:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829529.096611:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085851264:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829529.096614:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085851392:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829529.096622:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829529.096627:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829529/real 1787829529] req@ffff9e35c498e1c0 x1874672085851392/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829554 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829529.096633:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e1c0 x1874672085851392/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829554 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829529.096637:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829529.096639:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829529.096641:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085851392:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829529.223950:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829529.223969:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829529.224007:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:0.0:1787829529.224026:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829530.247890:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829530.247919:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829530.247920:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787829530.247942:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829531.271879:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829531.271893:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829531.271924:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829531.272009:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829532.295887:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829532.295890:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829532.295906:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:0.0:1787829532.295949:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829533.319873:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829533.319927:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787829533.319940:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829533.319950:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00080000:2.0:1787829534.215907:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829534.215917:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829534.215921:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829534.215934:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2969 00000100:00080000:2.0:1787829534.215945:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829534.215960:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787829534.215999:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f0c0 x1874672085851520/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829534.216032:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829534.216036:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829534.216038:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829534.216043:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2969 00000100:00080000:2.0:1787829534.216047:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829534.216052:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829534.216063:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829534.216081:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085851520:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1787829534.216089:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c000 x1874672085851648/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829534.216111:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829534.216116:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829534.216118:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829534.216124:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2969 00000100:00080000:2.0:1787829534.216131:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829534.216136:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829534.216142:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829534.216156:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cb40 x1874672085851776/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829534.216165:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829534.216168:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829534.216170:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829534.216177:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2969 00000100:00080000:2.0:1787829534.216181:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829534.216184:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829534.216188:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000800:00000100:0.0:1787829534.216195:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829534.216204:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498fc00 x1874672085851904/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829534.216214:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829534.216216:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829534.216218:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829534.216222:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2969 00000100:00080000:2.0:1787829534.216224:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829534.216227:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829534.216230:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:2.0:1787829534.216239:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e580 x1874672085852032/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829534.216248:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787829534.216254:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829534/real 1787829534] req@ffff9e35c498f0c0 x1874672085851520/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829559 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829534.216254:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829534.216260:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829534.216272:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2969 00000100:00080000:2.0:1787829534.216279:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829534.216286:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829534.216293:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829534.216304:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e940 x1874672085852160/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829534.216309:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f0c0 x1874672085851520/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829559 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829534.216316:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829534.216330:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829534.216336:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829534.216345:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2969 00000100:00080000:2.0:1787829534.216350:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829534.216355:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:0.0:1787829534.216357:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:2.0:1787829534.216358:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1787829534.216374:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829534.216375:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f480 x1874672085852288/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829534.216391:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787829534.216392:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085851520:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1787829534.216395:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829534.216401:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829534.216410:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2969 00000100:00080000:2.0:1787829534.216411:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829534.216415:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829534.216418:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:2.0:1787829534.216426:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e1c0 x1874672085852416/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829534.216437:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829534.216443:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:0.0:1787829534.216444:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085851648:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1787829534.216445:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829534.216454:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2969 00000100:00080000:2.0:1787829534.216458:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829534.216460:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829534.216466:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:2.0:1787829534.216476:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498d680 x1874672085852544/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829534.216494:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829534.216531:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829534/real 1787829534] req@ffff9e35c498c000 x1874672085851648/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829559 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829534.216557:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c000 x1874672085851648/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829559 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829534.216569:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829534.216574:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829534.216579:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085851648:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829534.216589:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085851776:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829534.216613:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829534.216636:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829534/real 1787829534] req@ffff9e35c498cb40 x1874672085851776/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829559 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829534.216654:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498cb40 x1874672085851776/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829559 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829534.216669:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829534.216677:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829534.216683:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085851776:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829534.216693:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085851904:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829534.216719:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829534.216731:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829534/real 1787829534] req@ffff9e35c498fc00 x1874672085851904/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829559 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829534.216745:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498fc00 x1874672085851904/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829559 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829534.216762:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829534.216766:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829534.216770:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085851904:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829534.216777:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085852032:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829534.216796:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829534.216844:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829534/real 1787829534] req@ffff9e35c498e580 x1874672085852032/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829559 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829534.216859:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e580 x1874672085852032/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829559 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829534.216870:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829534.216875:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829534.216881:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085852032:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829534.216889:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085852160:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829534.216911:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829534.216925:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829534/real 1787829534] req@ffff9e35c498e940 x1874672085852160/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829559 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829534.216939:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e940 x1874672085852160/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829559 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829534.216949:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829534.216952:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829534.216957:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085852160:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829534.216964:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085852288:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829534.216984:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829534.216995:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829534/real 1787829534] req@ffff9e35c498f480 x1874672085852288/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829559 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829534.217009:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f480 x1874672085852288/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829559 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829534.217018:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829534.217022:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829534.217025:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085852288:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829534.217051:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085852416:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829534.217072:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829534.217084:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829534/real 1787829534] req@ffff9e35c498e1c0 x1874672085852416/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829559 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829534.217098:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e1c0 x1874672085852416/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829559 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829534.217108:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829534.217112:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829534.217116:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085852416:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829534.217123:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085852544:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829534.217141:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829534.217162:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829534/real 1787829534] req@ffff9e35c498d680 x1874672085852544/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829559 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829534.217181:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498d680 x1874672085852544/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829559 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829534.217191:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829534.217195:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829534.217199:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085852544:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829534.343919:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829534.343921:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829534.343946:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829534.344004:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829535.367920:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829535.367920:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829535.367955:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787829535.367963:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829536.391898:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829536.391898:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829536.391937:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787829536.391967:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829537.415931:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829537.415937:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829537.415938:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787829537.416017:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829538.439897:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829538.439897:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829538.439903:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:0.0:1787829538.439995:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829539.335874:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829539.335880:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829539.335883:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829539.335891:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2974 00000100:00080000:2.0:1787829539.335901:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829539.335907:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787829539.335942:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c3c0 x1874672085852672/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829539.335970:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829539.335972:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829539.335973:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829539.335976:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2974 00000100:00080000:2.0:1787829539.335979:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829539.335982:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829539.335989:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829539.335998:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cf00 x1874672085852800/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829539.336005:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829539.336007:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829539.336008:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829539.336011:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2974 00000100:00080000:2.0:1787829539.336013:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829539.336015:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829539.336018:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829539.336026:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f840 x1874672085852928/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829539.336027:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085852672:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1787829539.336038:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829539.336040:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829539.336041:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829539.336043:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2974 00000100:00080000:2.0:1787829539.336045:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829539.336053:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829539.336055:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:2.0:1787829539.336063:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498ed00 x1874672085853056/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829539.336069:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829539.336071:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829539.336072:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829539.336075:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2974 00000100:00080000:2.0:1787829539.336077:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829539.336078:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829539.336081:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:2.0:1787829539.336088:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498de00 x1874672085853184/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829539.336093:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829539.336096:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000800:00000100:0.0:1787829539.336099:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1787829539.336099:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829539.336103:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2974 00000100:00080000:2.0:1787829539.336106:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829539.336108:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829539.336111:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829539.336117:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c780 x1874672085853312/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829539.336123:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829539.336124:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829539.336125:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829539.336127:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2974 00000100:00080000:2.0:1787829539.336129:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829539.336131:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829539.336134:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829539.336145:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498d680 x1874672085853440/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829539.336146:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829539/real 1787829539] req@ffff9e35c498c3c0 x1874672085852672/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829564 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829539.336155:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829539.336161:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829539.336163:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829539.336169:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2974 00000100:00080000:2.0:1787829539.336172:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829539.336175:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829539.336178:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:0.0:1787829539.336182:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c3c0 x1874672085852672/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829564 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829539.336186:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e1c0 x1874672085853568/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829539.336193:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829539.336197:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829539.336200:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829539.336207:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2974 00000100:00080000:0.0:1787829539.336209:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829539.336211:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829539.336218:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:2.0:1787829539.336219:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829539.336226:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:0.0:1787829539.336227:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085852672:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1787829539.336240:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f480 x1874672085853696/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829539.336241:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085852800:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829539.336275:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829539.336289:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829539/real 1787829539] req@ffff9e35c498cf00 x1874672085852800/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829564 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829539.336301:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498cf00 x1874672085852800/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829564 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829539.336309:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829539.336312:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829539.336315:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085852800:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829539.336340:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085852928:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829539.336360:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829539.336371:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829539/real 1787829539] req@ffff9e35c498f840 x1874672085852928/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829564 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829539.336382:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f840 x1874672085852928/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829564 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829539.336389:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829539.336392:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829539.336395:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085852928:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829539.336400:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085853056:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829539.336419:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829539.336430:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829539/real 1787829539] req@ffff9e35c498ed00 x1874672085853056/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829564 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829539.336446:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498ed00 x1874672085853056/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829564 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829539.336454:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829539.336457:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829539.336460:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085853056:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829539.336466:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085853184:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829539.336497:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829539.336507:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829539/real 1787829539] req@ffff9e35c498de00 x1874672085853184/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829564 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829539.336518:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498de00 x1874672085853184/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829564 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829539.336526:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829539.336529:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829539.336532:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085853184:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829539.336537:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085853312:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829539.336555:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829539.336565:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829539/real 1787829539] req@ffff9e35c498c780 x1874672085853312/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829564 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829539.336576:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c780 x1874672085853312/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829564 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829539.336588:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829539.336592:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829539.336594:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085853312:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829539.336600:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085853440:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829539.336618:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829539.336629:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829539/real 1787829539] req@ffff9e35c498d680 x1874672085853440/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829564 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829539.336640:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498d680 x1874672085853440/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829564 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829539.336647:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829539.336650:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829539.336653:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085853440:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829539.336658:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085853568:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829539.336675:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829539.336685:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829539/real 1787829539] req@ffff9e35c498e1c0 x1874672085853568/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829564 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829539.336696:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e1c0 x1874672085853568/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829564 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829539.336704:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829539.336706:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829539.336710:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085853568:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829539.336715:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085853696:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829539.336733:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829539.336743:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829539/real 1787829539] req@ffff9e35c498f480 x1874672085853696/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829564 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829539.336754:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f480 x1874672085853696/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829564 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829539.336761:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829539.336764:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829539.336767:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085853696:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829539.463863:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829539.463877:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829539.463885:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829539.463915:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829540.487879:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829540.487879:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829540.487891:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829540.487914:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829541.511901:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829541.511901:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829541.511901:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787829541.511974:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829542.535892:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829542.535896:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829542.535902:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829542.535953:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829543.559888:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829543.559893:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829543.559894:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829543.559925:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829544.455877:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829544.455884:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829544.455887:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829544.455894:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2979 00000100:00080000:1.0:1787829544.455903:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829544.455908:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787829544.455937:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7480 x1874672085853824/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829544.455962:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829544.455966:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829544.455968:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829544.455972:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2979 00000100:00080000:1.0:1787829544.455976:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829544.455979:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829544.455992:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829544.456005:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085853824:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1787829544.456005:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4000 x1874672085853952/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829544.456014:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829544.456017:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829544.456018:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829544.456021:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2979 00000100:00080000:1.0:1787829544.456024:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829544.456027:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829544.456031:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829544.456039:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6940 x1874672085854080/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829544.456047:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829544.456050:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829544.456051:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829544.456055:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2979 00000100:00080000:1.0:1787829544.456057:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1787829544.456067:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787829544.456069:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829544.456073:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:1.0:1787829544.456082:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6d00 x1874672085854208/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829544.456090:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829544.456092:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829544.456094:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829544.456097:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2979 00000100:00080000:1.0:1787829544.456100:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787829544.456103:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829544/real 1787829544] req@ffff9e35da6f7480 x1874672085853824/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829569 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829544.456103:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829544.456107:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:1.0:1787829544.456115:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4780 x1874672085854336/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829544.456119:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7480 x1874672085853824/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829569 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829544.456123:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829544.456127:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829544.456128:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829544.456129:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829544.456131:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2979 00000100:00080000:0.0:1787829544.456133:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:1.0:1787829544.456134:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787829544.456136:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085853824:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787829544.456137:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829544.456141:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829544.456150:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6580 x1874672085854464/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829544.456157:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085853952:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787829544.456157:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829544.456160:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829544.456162:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829544.456165:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2979 00000100:00080000:1.0:1787829544.456168:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829544.456171:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829544.456175:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1787829544.456176:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829544.456183:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4f00 x1874672085854592/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829544.456186:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829544/real 1787829544] req@ffff9e35da6f4000 x1874672085853952/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829569 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829544.456192:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829544.456195:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:0.0:1787829544.456197:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4000 x1874672085853952/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829569 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829544.456197:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829544.456201:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2979 00000100:00080000:1.0:1787829544.456203:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829544.456205:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829544.456207:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:0.0:1787829544.456208:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000008:00100000:1.0:1787829544.456211:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:0.0:1787829544.456212:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085853952:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829544.456217:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085854080:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829544.456219:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f61c0 x1874672085854720/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829544.456226:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829544.456229:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829544.456231:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1787829544.456232:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787829544.456235:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2979 00000100:00080000:1.0:1787829544.456238:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787829544.456241:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829544/real 1787829544] req@ffff9e35da6f6940 x1874672085854080/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829569 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829544.456241:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829544.456245:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:0.0:1787829544.456252:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6940 x1874672085854080/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829569 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829544.456254:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7840 x1874672085854848/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829544.456261:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829544.456263:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829544.456266:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085854080:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829544.456271:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085854208:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829544.456296:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829544.456303:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829544/real 1787829544] req@ffff9e35da6f6d00 x1874672085854208/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829569 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829544.456311:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6d00 x1874672085854208/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829569 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829544.456315:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829544.456317:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829544.456319:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085854208:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829544.456323:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085854336:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829544.456332:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829544.456337:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829544/real 1787829544] req@ffff9e35da6f4780 x1874672085854336/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829569 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829544.456343:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4780 x1874672085854336/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829569 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829544.456348:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829544.456349:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829544.456351:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085854336:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829544.456365:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085854464:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829544.456374:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829544.456380:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829544/real 1787829544] req@ffff9e35da6f6580 x1874672085854464/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829569 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829544.456391:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6580 x1874672085854464/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829569 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829544.456395:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829544.456397:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829544.456398:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085854464:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829544.456401:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085854592:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829544.456410:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829544.456415:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829544/real 1787829544] req@ffff9e35da6f4f00 x1874672085854592/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829569 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829544.456422:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4f00 x1874672085854592/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829569 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829544.456429:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829544.456432:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829544.456448:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085854592:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829544.456454:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085854720:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829544.456472:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829544.456482:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829544/real 1787829544] req@ffff9e35da6f61c0 x1874672085854720/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829569 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829544.456490:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f61c0 x1874672085854720/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829569 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829544.456498:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829544.456500:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829544.456502:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085854720:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829544.456505:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085854848:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829544.456516:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829544.456522:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829544/real 1787829544] req@ffff9e35da6f7840 x1874672085854848/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829569 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829544.456528:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7840 x1874672085854848/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829569 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829544.456533:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829544.456534:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829544.456536:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085854848:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829544.583872:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829544.583892:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:3.0:1787829544.583956:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787829544.584010:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829545.607907:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829545.607907:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829545.607936:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829545.607948:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829546.631886:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829546.631911:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829546.631913:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787829546.631951:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829547.655898:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829547.655898:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829547.655907:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:0.0:1787829547.655969:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829548.679876:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:3.0:1787829548.679883:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829548.679887:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829548.679945:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829549.575890:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829549.575906:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787829549.575910:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829549.575919:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2984 00000100:00080000:0.0:1787829549.575928:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829549.575935:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787829549.575968:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718780 x1874672085854976/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829549.575993:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829549.575995:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829549.575997:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829549.576000:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2984 00000100:00080000:0.0:1787829549.576002:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829549.576005:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829549.576013:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829549.576030:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718f00 x1874672085855104/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829549.576033:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085854976:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1787829549.576046:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829549.576049:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787829549.576050:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829549.576053:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2984 00000100:00080000:0.0:1787829549.576055:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829549.576058:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829549.576061:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829549.576070:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a940 x1874672085855232/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829549.576077:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829549.576079:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829549.576081:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829549.576084:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2984 00000100:00080000:0.0:1787829549.576089:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829549.576092:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829549.576096:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000800:00000100:1.0:1787829549.576100:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829549.576108:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7183c0 x1874672085855360/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829549.576115:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829549.576118:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829549.576120:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829549.576122:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2984 00000100:00080000:0.0:1787829549.576125:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829549.576127:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829549.576130:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:0.0:1787829549.576150:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da719a40 x1874672085855488/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829549.576150:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829549/real 1787829549] req@ffff9e35da718780 x1874672085854976/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829574 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829549.576178:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829549.576181:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829549.576183:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787829549.576189:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718780 x1874672085854976/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829574 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829549.576192:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2984 00000100:00080000:0.0:1787829549.576197:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829549.576203:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829549.576209:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00080000:1.0:1787829549.576210:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829549.576217:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829549.576222:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a580 x1874672085855616/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829549.576222:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085854976:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1787829549.576235:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829549.576237:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787829549.576238:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829549.576242:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2984 00000100:00080000:0.0:1787829549.576244:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829549.576247:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829549.576250:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829549.576252:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085855104:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829549.576265:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718000 x1874672085855744/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829549.576274:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1787829549.576278:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787829549.576279:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829549.576282:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829549.576287:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2984 00000100:00080000:0.0:1787829549.576290:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1787829549.576294:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829549/real 1787829549] req@ffff9e35da718f00 x1874672085855104/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829574 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829549.576295:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829549.576301:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:0.0:1787829549.576313:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718b40 x1874672085855872/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829549.576315:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672085855104/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829574 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829549.576333:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829549.576336:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829549.576337:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829549.576341:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829549.576344:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:0.0:1787829549.576348:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2984 00000100:00100000:1.0:1787829549.576352:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085855104:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787829549.576353:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829549.576360:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829549.576363:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:1.0:1787829549.576364:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085855232:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829549.576372:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a1c0 x1874672085856000/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787829549.576383:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829549.576393:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829549/real 1787829549] req@ffff9e35da71a940 x1874672085855232/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829574 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829549.576422:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672085855232/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829574 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829549.576428:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829549.576430:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829549.576433:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085855232:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829549.576437:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085855360:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829549.576460:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829549.576467:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829549/real 1787829549] req@ffff9e35da7183c0 x1874672085855360/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829574 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829549.576475:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672085855360/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829574 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829549.576481:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829549.576483:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829549.576485:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085855360:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829549.576489:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085855488:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829549.576500:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829549.576507:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829549/real 1787829549] req@ffff9e35da719a40 x1874672085855488/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829574 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829549.576514:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672085855488/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829574 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829549.576519:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829549.576521:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829549.576523:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085855488:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829549.576527:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085855616:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787829549.576537:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829549.576543:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829549/real 1787829549] req@ffff9e35da71a580 x1874672085855616/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829574 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829549.576550:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672085855616/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829574 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829549.576561:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829549.576563:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829549.576565:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085855616:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829549.576580:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085855744:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787829549.576592:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829549.576598:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829549/real 1787829549] req@ffff9e35da718000 x1874672085855744/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829574 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829549.576606:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718000 x1874672085855744/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829574 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829549.576611:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829549.576613:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829549.576615:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085855744:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829549.576619:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085855872:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829549.576628:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829549.576633:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829549/real 1787829549] req@ffff9e35da718b40 x1874672085855872/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829574 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829549.576640:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718b40 x1874672085855872/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829574 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829549.576646:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829549.576648:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829549.576650:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085855872:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829549.576654:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085856000:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829549.576669:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829549.576676:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829549/real 1787829549] req@ffff9e35da71a1c0 x1874672085856000/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829574 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829549.576683:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a1c0 x1874672085856000/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829574 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829549.576688:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829549.576690:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829549.576692:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085856000:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829549.703902:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829549.703903:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829549.704019:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787829549.704098:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829550.727887:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829550.727892:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829550.727915:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:0.0:1787829550.727950:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829551.751898:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829551.751898:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829551.751916:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829551.751932:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829552.775866:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829552.775868:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:0.0:1787829552.776034:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829552.776061:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829553.799915:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829553.799915:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829553.799941:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787829553.799981:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829554.695909:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829554.695918:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787829554.695921:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829554.695929:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2990 00000100:00080000:0.0:1787829554.695939:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829554.695944:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787829554.695977:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7192c0 x1874672085856128/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829554.696011:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829554.696016:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829554.696018:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829554.696023:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2990 00000100:00080000:0.0:1787829554.696027:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829554.696033:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829554.696044:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829554.696061:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a1c0 x1874672085856256/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829554.696070:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829554.696074:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787829554.696078:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787829554.696078:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085856128:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1787829554.696090:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2990 00000100:00080000:0.0:1787829554.696094:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829554.696097:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829554.696101:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829554.696110:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718b40 x1874672085856384/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829554.696118:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829554.696120:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829554.696121:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829554.696124:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2990 00000100:00080000:0.0:1787829554.696138:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829554.696142:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829554.696147:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000800:00000100:1.0:1787829554.696150:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829554.696163:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718000 x1874672085856512/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829554.696175:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829554.696179:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829554.696182:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829554.696187:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2990 00000100:00080000:0.0:1787829554.696192:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829554.696196:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829554.696202:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:1.0:1787829554.696212:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829554/real 1787829554] req@ffff9e35da7192c0 x1874672085856128/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829579 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829554.696218:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a580 x1874672085856640/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829554.696241:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829554.696244:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829554.696246:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787829554.696248:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7192c0 x1874672085856128/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829579 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829554.696252:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2990 00000100:00080000:0.0:1787829554.696257:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829554.696263:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829554.696270:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00080000:1.0:1787829554.696273:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829554.696281:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829554.696287:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085856128:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1787829554.696289:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da719a40 x1874672085856768/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829554.696302:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829554.696306:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787829554.696308:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829554.696313:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2990 00000100:00080000:0.0:1787829554.696317:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1787829554.696321:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085856256:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787829554.696326:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829554.696334:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829554.696351:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7183c0 x1874672085856896/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787829554.696354:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787829554.696381:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787829554.696386:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829554/real 1787829554] req@ffff9e35da71a1c0 x1874672085856256/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829579 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829554.696387:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829554.696391:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829554.696399:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2990 00000100:00080000:0.0:1787829554.696403:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1787829554.696406:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a1c0 x1874672085856256/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829579 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829554.696410:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829554.696418:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00080000:1.0:1787829554.696424:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829554.696431:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829554.696433:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a940 x1874672085857024/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829554.696439:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085856256:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787829554.696453:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787829554.696455:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085856384:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787829554.696459:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829554.696464:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829554.696469:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2990 00000100:00080000:0.0:1787829554.696474:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829554.696479:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829554.696485:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000800:00000100:1.0:1787829554.696490:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829554.696503:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718f00 x1874672085857152/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829554.696509:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829554/real 1787829554] req@ffff9e35da718b40 x1874672085856384/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829579 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829554.696532:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718b40 x1874672085856384/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829579 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829554.696542:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829554.696546:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829554.696549:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085856384:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829554.696556:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085856512:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829554.696579:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829554.696591:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829554/real 1787829554] req@ffff9e35da718000 x1874672085856512/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829579 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829554.696603:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718000 x1874672085856512/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829579 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829554.696612:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829554.696615:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829554.696618:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085856512:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829554.696625:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085856640:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829554.696646:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829554.696658:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829554/real 1787829554] req@ffff9e35da71a580 x1874672085856640/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829579 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829554.696671:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672085856640/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829579 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829554.696681:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829554.696684:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829554.696687:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085856640:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829554.696694:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085856768:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787829554.696717:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829554.696729:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829554/real 1787829554] req@ffff9e35da719a40 x1874672085856768/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829579 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829554.696742:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672085856768/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829579 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829554.696751:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829554.696754:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829554.696758:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085856768:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829554.696781:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085856896:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787829554.696828:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829554.696841:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829554/real 1787829554] req@ffff9e35da7183c0 x1874672085856896/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829579 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829554.696861:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672085856896/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829579 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829554.696869:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829554.696873:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829554.696876:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085856896:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829554.696883:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085857024:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829554.696905:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829554.696917:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829554/real 1787829554] req@ffff9e35da71a940 x1874672085857024/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829579 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829554.696930:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672085857024/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829579 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829554.696939:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829554.696943:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829554.696946:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085857024:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829554.696953:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085857152:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829554.696972:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829554.696983:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829554/real 1787829554] req@ffff9e35da718f00 x1874672085857152/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829579 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829554.696994:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672085857152/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829579 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829554.697009:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829554.697012:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829554.697016:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085857152:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829554.823850:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829554.823873:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829554.823876:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829554.823937:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829555.847874:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829555.847907:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787829555.847918:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829555.848001:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829556.871884:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787829556.871884:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829556.871937:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787829556.871996:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829557.895882:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829557.895885:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829557.895905:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:0.0:1787829557.895927:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829558.919856:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829558.919873:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829558.919903:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787829558.919914:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829559.815953:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829559.815963:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829559.815967:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829559.815978:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2995 00000100:00080000:1.0:1787829559.815991:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829559.815999:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787829559.816040:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f43c0 x1874672085857280/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829559.816077:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829559.816081:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829559.816083:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829559.816088:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2995 00000100:00080000:1.0:1787829559.816091:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829559.816094:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829559.816103:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829559.816105:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085857280:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1787829559.816119:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f70c0 x1874672085857408/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829559.816133:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829559.816137:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829559.816139:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829559.816144:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2995 00000100:00080000:1.0:1787829559.816159:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829559.816164:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:0.0:1787829559.816168:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:1.0:1787829559.816169:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829559.816183:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4b40 x1874672085857536/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829559.816191:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829559.816193:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829559.816195:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829559.816198:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2995 00000100:00080000:1.0:1787829559.816202:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829559.816205:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787829559.816208:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829559/real 1787829559] req@ffff9e35da6f43c0 x1874672085857280/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829584 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1787829559.816210:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:1.0:1787829559.816223:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5e00 x1874672085857664/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829559.816229:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f43c0 x1874672085857280/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829584 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829559.816235:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829559.816242:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829559.816245:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829559.816248:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829559.816249:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 2995 00000100:00080000:1.0:1787829559.816254:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829559.816255:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:1.0:1787829559.816261:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787829559.816263:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085857280:192.168.203.141@tcp:250:kworker.0 00000008:00100000:1.0:1787829559.816266:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:1.0:1787829559.816275:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5a40 x1874672085857792/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829559.816282:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829559.816284:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:0.0:1787829559.816287:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085857408:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787829559.816288:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829559.816297:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2995 00000100:00080000:1.0:1787829559.816300:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829559.816302:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829559.816306:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1787829559.816314:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829559.816318:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f43c0 x1874672085857920/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829559.816339:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829559/real 1787829559] req@ffff9e35da6f70c0 x1874672085857408/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829584 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829559.816342:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829559.816350:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829559.816354:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829559.816360:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2995 00000100:00100000:0.0:1787829559.816364:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f70c0 x1874672085857408/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829584 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829559.816364:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829559.816374:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:0.0:1787829559.816379:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000008:00100000:1.0:1787829559.816383:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1787829559.816384:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829559.816390:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085857408:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829559.816395:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085857536:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829559.816403:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7840 x1874672085858048/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829559.816418:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787829559.816428:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787829559.816430:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829559/real 1787829559] req@ffff9e35da6f4b40 x1874672085857536/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829584 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829559.816438:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829559.816442:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829559.816451:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2995 00000100:00100000:0.0:1787829559.816453:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4b40 x1874672085857536/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829584 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829559.816456:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829559.816464:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:0.0:1787829559.816466:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000008:00100000:1.0:1787829559.816469:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00080000:0.0:1787829559.816471:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829559.816477:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085857536:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829559.816485:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f61c0 x1874672085858176/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829559.816492:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085857664:192.168.203.141@tcp:8:kworker.0 00000100:00080000:1.0:1787829559.816494:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829559.816500:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829559.816502:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829559.816505:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 2995 00000800:00000100:0.0:1787829559.816513:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787829559.816518:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787829559.816525:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829559/real 1787829559] req@ffff9e35da6f5e00 x1874672085857664/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829584 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829559.816528:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829559.816535:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:0.0:1787829559.816540:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5e00 x1874672085857664/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829584 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829559.816547:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4f00 x1874672085858304/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829559.816550:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829559.816559:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829559.816565:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085857664:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829559.816570:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085857792:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829559.816584:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829559.816592:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829559/real 1787829559] req@ffff9e35da6f5a40 x1874672085857792/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829584 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829559.816604:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5a40 x1874672085857792/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829584 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829559.816609:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829559.816611:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829559.816613:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085857792:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829559.816630:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085857920:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829559.816646:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829559.816657:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829559/real 1787829559] req@ffff9e35da6f43c0 x1874672085857920/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829584 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829559.816669:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f43c0 x1874672085857920/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829584 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829559.816678:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829559.816681:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829559.816685:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085857920:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829559.816691:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085858048:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829559.816707:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829559.816714:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829559/real 1787829559] req@ffff9e35da6f7840 x1874672085858048/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829584 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829559.816722:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7840 x1874672085858048/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829584 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829559.816731:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829559.816733:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829559.816735:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085858048:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829559.816739:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085858176:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829559.816748:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829559.816754:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829559/real 1787829559] req@ffff9e35da6f61c0 x1874672085858176/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829584 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829559.816760:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f61c0 x1874672085858176/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829584 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829559.816766:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829559.816768:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829559.816770:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085858176:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829559.816774:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085858304:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829559.816782:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829559.816787:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829559/real 1787829559] req@ffff9e35da6f4f00 x1874672085858304/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829584 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829559.816794:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4f00 x1874672085858304/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829584 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829559.816818:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829559.816820:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829559.816822:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085858304:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829559.943884:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829559.943893:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829559.943907:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787829559.943914:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829560.967927:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829560.967927:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829560.967960:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787829560.968029:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829561.991935:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829561.991960:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787829561.992000:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829561.992008:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829563.015934:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829563.015934:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829563.015966:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787829563.015989:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829564.039907:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829564.039916:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829564.039936:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829564.039939:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829564.935912:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829564.935923:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787829564.935927:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829564.935938:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3000 00000100:00080000:0.0:1787829564.935949:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829564.935955:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787829564.936007:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718780 x1874672085858432/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829564.936044:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829564.936048:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829564.936051:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829564.936056:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3000 00000100:00080000:0.0:1787829564.936060:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829564.936064:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829564.936075:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829564.936089:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718f00 x1874672085858560/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829564.936099:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085858432:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1787829564.936100:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829564.936108:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787829564.936111:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829564.936114:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3000 00000100:00080000:0.0:1787829564.936117:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829564.936119:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829564.936122:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829564.936131:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a940 x1874672085858688/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829564.936138:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829564.936141:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829564.936142:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829564.936145:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3000 00000100:00080000:0.0:1787829564.936147:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829564.936149:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829564.936152:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:0.0:1787829564.936159:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7183c0 x1874672085858816/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829564.936165:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829564.936168:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829564.936170:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829564.936172:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3000 00000100:00080000:0.0:1787829564.936175:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829564.936177:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829564.936182:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000800:00000100:1.0:1787829564.936183:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829564.936191:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da719a40 x1874672085858944/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829564.936199:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829564.936201:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829564.936203:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829564.936206:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3000 00000100:00080000:0.0:1787829564.936210:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829564.936213:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829564.936215:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829564.936223:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a580 x1874672085859072/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829564.936230:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829564.936233:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787829564.936235:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787829564.936239:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829564/real 1787829564] req@ffff9e35da718780 x1874672085858432/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829589 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829564.936240:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3000 00000100:00080000:0.0:1787829564.936244:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829564.936249:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829564.936255:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829564.936266:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718000 x1874672085859200/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829564.936273:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829564.936277:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:1.0:1787829564.936277:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718780 x1874672085858432/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829589 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829564.936279:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829564.936286:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3000 00000100:00080000:0.0:1787829564.936290:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829564.936310:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:1.0:1787829564.936314:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:0.0:1787829564.936315:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00080000:1.0:1787829564.936324:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829564.936329:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718b40 x1874672085859328/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829564.936334:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085858432:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1787829564.936340:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829564.936344:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829564.936346:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829564.936350:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3000 00000100:00080000:0.0:1787829564.936352:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829564.936355:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829564.936358:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:0.0:1787829564.936365:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a1c0 x1874672085859456/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829564.936372:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085858560:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787829564.936402:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829564.936414:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829564/real 1787829564] req@ffff9e35da718f00 x1874672085858560/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829589 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829564.936424:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672085858560/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829589 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829564.936431:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829564.936433:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829564.936436:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085858560:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829564.936441:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085858688:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787829564.936454:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829564.936464:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829564/real 1787829564] req@ffff9e35da71a940 x1874672085858688/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829589 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829564.936485:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672085858688/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829589 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829564.936494:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829564.936497:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829564.936501:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085858688:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829564.936508:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085858816:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829564.936531:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829564.936543:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829564/real 1787829564] req@ffff9e35da7183c0 x1874672085858816/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829589 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829564.936556:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672085858816/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829589 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829564.936563:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829564.936566:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829564.936568:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085858816:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829564.936573:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085858944:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829564.936588:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829564.936596:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829564/real 1787829564] req@ffff9e35da719a40 x1874672085858944/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829589 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829564.936603:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672085858944/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829589 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829564.936614:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829564.936616:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829564.936618:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085858944:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829564.936622:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085859072:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787829564.936632:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829564.936638:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829564/real 1787829564] req@ffff9e35da71a580 x1874672085859072/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829589 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829564.936646:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672085859072/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829589 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829564.936651:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829564.936654:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829564.936656:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085859072:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829564.936659:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085859200:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787829564.936669:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829564.936674:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829564/real 1787829564] req@ffff9e35da718000 x1874672085859200/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829589 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829564.936681:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718000 x1874672085859200/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829589 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829564.936698:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829564.936702:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829564.936706:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085859200:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829564.936713:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085859328:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829564.936734:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829564.936754:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829564/real 1787829564] req@ffff9e35da718b40 x1874672085859328/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829589 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829564.936768:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718b40 x1874672085859328/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829589 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829564.936777:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829564.936781:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829564.936785:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085859328:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829564.936835:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085859456:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829564.936853:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829564.936861:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829564/real 1787829564] req@ffff9e35da71a1c0 x1874672085859456/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829589 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829564.936870:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a1c0 x1874672085859456/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829589 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829564.936880:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829564.936883:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829564.936887:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085859456:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829565.063919:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829565.063919:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829565.063936:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787829565.063971:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829566.087909:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829566.087909:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829566.087945:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787829566.087956:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829567.111928:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829567.111950:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829567.111961:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:3.0:1787829567.111993:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829568.135935:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829568.135948:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787829568.135948:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787829568.136076:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829569.159910:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829569.159919:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829569.159956:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:0.0:1787829569.159981:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829570.055896:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829570.055905:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787829570.055909:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829570.055921:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3005 00000100:00080000:0.0:1787829570.055933:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829570.055942:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787829570.055978:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7192c0 x1874672085859584/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829570.056008:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829570.056012:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829570.056014:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829570.056018:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3005 00000100:00080000:0.0:1787829570.056022:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829570.056026:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829570.056036:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829570.056048:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a1c0 x1874672085859712/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829570.056053:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085859584:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1787829570.056059:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829570.056064:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787829570.056065:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829570.056069:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3005 00000100:00080000:0.0:1787829570.056072:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829570.056075:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829570.056079:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829570.056090:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718b40 x1874672085859840/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829570.056102:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829570.056105:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829570.056108:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829570.056112:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3005 00000100:00080000:0.0:1787829570.056115:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829570.056118:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829570.056122:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:0.0:1787829570.056131:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718000 x1874672085859968/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829570.056136:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787829570.056144:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829570.056147:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829570.056149:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829570.056153:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3005 00000100:00080000:0.0:1787829570.056158:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829570.056161:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829570.056167:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:0.0:1787829570.056177:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a580 x1874672085860096/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829570.056180:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829570/real 1787829570] req@ffff9e35da7192c0 x1874672085859584/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829595 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829570.056195:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829570.056198:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829570.056200:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829570.056204:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3005 00000100:00100000:2.0:1787829570.056208:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7192c0 x1874672085859584/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829595 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829570.056210:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829570.056217:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829570.056225:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1787829570.056225:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829570.056235:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829570.056240:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da719a40 x1874672085860224/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829570.056242:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085859584:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1787829570.056276:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829570.056279:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787829570.056281:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829570.056287:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3005 00000100:00080000:0.0:1787829570.056290:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829570.056293:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829570.056298:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829570.056301:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085859712:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829570.056311:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7183c0 x1874672085860352/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829570.056321:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829570.056323:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829570.056325:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1787829570.056329:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787829570.056330:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3005 00000100:00080000:0.0:1787829570.056337:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829570.056341:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829570.056345:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:2.0:1787829570.056349:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829570/real 1787829570] req@ffff9e35da71a1c0 x1874672085859712/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829595 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829570.056357:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a940 x1874672085860480/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829570.056365:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a1c0 x1874672085859712/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829595 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829570.056388:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829570.056395:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:0.0:1787829570.056398:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787829570.056399:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085859712:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787829570.056408:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829570.056410:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787829570.056412:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085859840:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787829570.056418:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3005 00000100:00080000:0.0:1787829570.056422:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829570.056426:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829570.056430:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000800:00000100:2.0:1787829570.056434:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829570.056448:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829570/real 1787829570] req@ffff9e35da718b40 x1874672085859840/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829595 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829570.056449:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718f00 x1874672085860608/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829570.056483:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718b40 x1874672085859840/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829595 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829570.056490:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829570.056494:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829570.056497:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085859840:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829570.056505:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085859968:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829570.056530:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829570.056546:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829570/real 1787829570] req@ffff9e35da718000 x1874672085859968/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829595 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829570.056563:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718000 x1874672085859968/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829595 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829570.056574:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829570.056579:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829570.056583:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085859968:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829570.056589:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085860096:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829570.056615:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829570.056625:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829570/real 1787829570] req@ffff9e35da71a580 x1874672085860096/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829595 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829570.056632:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672085860096/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829595 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829570.056639:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829570.056641:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829570.056643:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085860096:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829570.056647:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085860224:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787829570.056658:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829570.056664:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829570/real 1787829570] req@ffff9e35da719a40 x1874672085860224/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829595 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829570.056671:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672085860224/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829595 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829570.056680:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829570.056682:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829570.056684:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085860224:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829570.056702:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085860352:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787829570.056720:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829570.056732:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829570/real 1787829570] req@ffff9e35da7183c0 x1874672085860352/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829595 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829570.056745:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672085860352/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829595 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829570.056754:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829570.056757:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829570.056760:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085860352:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829570.056773:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085860480:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829570.056789:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829570.056819:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829570/real 1787829570] req@ffff9e35da71a940 x1874672085860480/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829595 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829570.056829:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672085860480/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829595 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829570.056836:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829570.056838:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829570.056841:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085860480:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829570.056846:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085860608:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829570.056863:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829570.056870:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829570/real 1787829570] req@ffff9e35da718f00 x1874672085860608/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829595 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829570.056877:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672085860608/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829595 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829570.056883:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829570.056885:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829570.056887:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085860608:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829570.183920:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829570.183931:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829570.183976:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787829570.184009:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829571.207913:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829571.207913:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829571.207940:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787829571.207961:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829572.231871:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829572.231871:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829572.231901:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829572.231948:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829573.255894:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829573.255894:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787829573.255947:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829573.256050:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829574.279965:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829574.279975:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829574.280007:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829574.280027:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829575.175887:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829575.175898:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787829575.175903:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829575.175914:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3010 00000100:00080000:0.0:1787829575.175926:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829575.175934:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787829575.175972:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718780 x1874672085860736/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829575.175995:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829575.175997:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829575.175999:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829575.176002:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3010 00000100:00080000:0.0:1787829575.176005:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829575.176009:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829575.176018:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829575.176030:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718f00 x1874672085860864/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829575.176038:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829575.176042:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:2.0:1787829575.176042:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085860736:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1787829575.176043:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829575.176051:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3010 00000100:00080000:0.0:1787829575.176055:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829575.176057:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829575.176060:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829575.176069:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a940 x1874672085860992/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829575.176076:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829575.176078:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829575.176080:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829575.176082:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3010 00000100:00080000:0.0:1787829575.176084:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829575.176086:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829575.176090:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:0.0:1787829575.176097:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7183c0 x1874672085861120/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829575.176107:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829575.176110:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829575.176111:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829575.176113:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3010 00000100:00080000:0.0:1787829575.176116:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829575.176122:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:2.0:1787829575.176122:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:0.0:1787829575.176129:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:0.0:1787829575.176137:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da719a40 x1874672085861248/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829575.176143:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829575.176146:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829575.176147:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829575.176150:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3010 00000100:00080000:0.0:1787829575.176153:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829575.176155:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829575.176158:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829575.176167:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a580 x1874672085861376/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829575.176172:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829575/real 1787829575] req@ffff9e35da718780 x1874672085860736/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829600 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829575.176176:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829575.176182:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787829575.176184:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829575.176192:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3010 00000100:00080000:0.0:1787829575.176197:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829575.176200:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829575.176203:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829575.176211:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718780 x1874672085860736/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829600 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829575.176212:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718000 x1874672085861504/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829575.176255:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829575.176258:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829575.176258:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829575.176262:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829575.176266:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:0.0:1787829575.176270:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3010 00000100:00100000:2.0:1787829575.176273:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085860736:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1787829575.176276:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829575.176284:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829575.176290:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:2.0:1787829575.176298:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085860864:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829575.176306:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718b40 x1874672085861632/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829575.176320:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1787829575.176325:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787829575.176328:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829575.176332:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829575.176338:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3010 00000100:00100000:2.0:1787829575.176341:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829575/real 1787829575] req@ffff9e35da718f00 x1874672085860864/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829600 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829575.176345:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829575.176356:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787829575.176358:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672085860864/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829600 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1787829575.176362:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00080000:2.0:1787829575.176369:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829575.176373:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829575.176381:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a1c0 x1874672085861760/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829575.176381:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085860864:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829575.176390:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085860992:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787829575.176412:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829575.176423:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829575/real 1787829575] req@ffff9e35da71a940 x1874672085860992/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829600 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829575.176431:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672085860992/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829600 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829575.176437:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829575.176439:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829575.176441:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085860992:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829575.176445:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085861120:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829575.176457:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829575.176463:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829575/real 1787829575] req@ffff9e35da7183c0 x1874672085861120/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829600 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829575.176470:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672085861120/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829600 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829575.176476:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829575.176477:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829575.176479:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085861120:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829575.176483:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085861248:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829575.176492:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829575.176498:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829575/real 1787829575] req@ffff9e35da719a40 x1874672085861248/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829600 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829575.176509:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672085861248/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829600 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829575.176518:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829575.176522:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829575.176526:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085861248:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829575.176532:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085861376:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787829575.176554:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829575.176568:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829575/real 1787829575] req@ffff9e35da71a580 x1874672085861376/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829600 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829575.176581:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672085861376/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829600 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829575.176590:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829575.176593:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829575.176597:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085861376:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829575.176603:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085861504:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787829575.176623:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829575.176634:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829575/real 1787829575] req@ffff9e35da718000 x1874672085861504/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829600 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829575.176647:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718000 x1874672085861504/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829600 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829575.176655:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829575.176658:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829575.176661:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085861504:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829575.176683:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085861632:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829575.176701:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829575.176709:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829575/real 1787829575] req@ffff9e35da718b40 x1874672085861632/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829600 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829575.176716:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718b40 x1874672085861632/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829600 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829575.176727:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829575.176729:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829575.176731:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085861632:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829575.176735:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085861760:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829575.176746:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829575.176752:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829575/real 1787829575] req@ffff9e35da71a1c0 x1874672085861760/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829600 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829575.176759:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a1c0 x1874672085861760/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829600 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829575.176764:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829575.176766:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829575.176767:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085861760:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829575.303907:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829575.303926:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829575.303926:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:1.0:1787829575.303949:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829576.327947:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829576.327954:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829576.327972:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787829576.327998:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829577.351888:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829577.351929:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829577.351942:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829577.352018:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829578.375962:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829578.375975:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829578.375976:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787829578.376044:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829579.399890:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829579.399890:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829579.399919:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787829579.399939:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829580.295899:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829580.295909:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829580.295914:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829580.295926:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3015 00000100:00080000:1.0:1787829580.295937:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829580.295946:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787829580.295984:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6580 x1874672085861888/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829580.296007:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829580.296011:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829580.296013:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829580.296018:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3015 00000100:00080000:1.0:1787829580.296023:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829580.296028:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829580.296039:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829580.296053:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085861888:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1787829580.296056:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4780 x1874672085862016/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829580.296070:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829580.296072:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829580.296073:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829580.296076:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3015 00000100:00080000:1.0:1787829580.296079:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829580.296082:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829580.296087:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829580.296096:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6d00 x1874672085862144/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829580.296103:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829580.296106:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829580.296107:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829580.296109:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3015 00000100:00080000:1.0:1787829580.296111:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829580.296116:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:2.0:1787829580.296117:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:1.0:1787829580.296121:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:1.0:1787829580.296131:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6940 x1874672085862272/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829580.296142:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829580.296145:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829580.296146:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829580.296150:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3015 00000100:00080000:1.0:1787829580.296152:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829580.296158:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787829580.296159:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829580/real 1787829580] req@ffff9e35da6f6580 x1874672085861888/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829605 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1787829580.296161:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:1.0:1787829580.296172:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4000 x1874672085862400/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829580.296203:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829580.296206:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829580.296208:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787829580.296212:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6580 x1874672085861888/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829605 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829580.296213:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3015 00000100:00080000:1.0:1787829580.296218:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829580.296225:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:2.0:1787829580.296230:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:1.0:1787829580.296231:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1787829580.296237:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829580.296242:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085861888:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1787829580.296244:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7480 x1874672085862528/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829580.296253:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829580.296255:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829580.296256:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829580.296259:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3015 00000100:00080000:1.0:1787829580.296262:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829580.296264:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787829580.296269:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085862016:192.168.203.141@tcp:38:kworker.0 00000008:00100000:1.0:1787829580.296277:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829580.296289:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6580 x1874672085862656/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829580.296296:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1787829580.296299:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787829580.296301:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829580.296304:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829580.296309:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3015 00000100:00100000:2.0:1787829580.296316:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829580/real 1787829580] req@ffff9e35da6f4780 x1874672085862016/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829605 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829580.296320:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829580.296327:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787829580.296331:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4780 x1874672085862016/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829605 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1787829580.296333:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00080000:2.0:1787829580.296345:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1787829580.296348:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4f00 x1874672085862784/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829580.296351:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829580.296358:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085862016:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787829580.296364:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829580.296369:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:2.0:1787829580.296369:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085862144:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787829580.296373:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829580.296379:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3015 00000100:00080000:1.0:1787829580.296382:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829580.296385:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829580.296388:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:1.0:1787829580.296394:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f61c0 x1874672085862912/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829580.296404:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829580.296416:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829580/real 1787829580] req@ffff9e35da6f6d00 x1874672085862144/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829605 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829580.296425:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6d00 x1874672085862144/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829605 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829580.296435:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829580.296438:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829580.296440:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085862144:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829580.296444:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085862272:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829580.296460:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829580.296470:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829580/real 1787829580] req@ffff9e35da6f6940 x1874672085862272/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829605 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829580.296482:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6940 x1874672085862272/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829605 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829580.296500:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829580.296504:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829580.296507:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085862272:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829580.296513:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085862400:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829580.296529:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829580.296537:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829580/real 1787829580] req@ffff9e35da6f4000 x1874672085862400/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829605 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829580.296545:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4000 x1874672085862400/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829605 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829580.296551:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829580.296553:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829580.296555:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085862400:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829580.296559:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085862528:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787829580.296569:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829580.296575:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829580/real 1787829580] req@ffff9e35da6f7480 x1874672085862528/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829605 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829580.296581:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7480 x1874672085862528/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829605 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829580.296586:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829580.296588:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829580.296590:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085862528:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829580.296607:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085862656:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787829580.296617:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829580.296623:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829580/real 1787829580] req@ffff9e35da6f6580 x1874672085862656/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829605 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829580.296631:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6580 x1874672085862656/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829605 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829580.296636:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829580.296638:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829580.296640:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085862656:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829580.296643:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085862784:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829580.296652:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829580.296658:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829580/real 1787829580] req@ffff9e35da6f4f00 x1874672085862784/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829605 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829580.296665:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4f00 x1874672085862784/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829605 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829580.296671:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829580.296672:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829580.296674:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085862784:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829580.296677:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085862912:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829580.296686:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829580.296691:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829580/real 1787829580] req@ffff9e35da6f61c0 x1874672085862912/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829605 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829580.296699:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f61c0 x1874672085862912/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829605 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829580.296704:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829580.296705:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829580.296707:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085862912:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829580.423898:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829580.423899:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829580.423916:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787829580.423944:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829581.447871:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829581.447895:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829581.447981:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787829581.448023:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829582.471942:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829582.471948:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787829582.471978:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829582.471981:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829583.495899:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:3.0:1787829583.495901:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829583.495905:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787829583.495989:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829584.519896:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829584.519897:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829584.519924:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829584.519936:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829585.415982:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829585.415990:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829585.415993:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829585.416001:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3020 00000100:00080000:2.0:1787829585.416010:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829585.416016:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787829585.416048:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e940 x1874672085863040/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829585.416079:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829585.416084:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829585.416086:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829585.416090:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3020 00000100:00080000:2.0:1787829585.416096:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829585.416101:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829585.416112:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829585.416124:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e580 x1874672085863168/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829585.416135:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829585.416143:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829585.416146:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787829585.416147:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085863040:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1787829585.416170:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3020 00000100:00080000:2.0:1787829585.416178:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829585.416182:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829585.416187:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829585.416199:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498fc00 x1874672085863296/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829585.416207:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829585.416209:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829585.416211:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829585.416215:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3020 00000100:00080000:2.0:1787829585.416218:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829585.416221:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829585.416225:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:2.0:1787829585.416233:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cb40 x1874672085863424/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829585.416239:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829585.416242:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829585.416243:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829585.416246:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3020 00000100:00080000:2.0:1787829585.416248:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829585.416250:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829585.416254:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:2.0:1787829585.416262:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c000 x1874672085863552/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829585.416268:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829585.416271:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829585.416272:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829585.416276:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3020 00000100:00080000:2.0:1787829585.416278:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829585.416281:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829585.416285:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829585.416292:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f0c0 x1874672085863680/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829585.416297:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829585.416300:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829585.416302:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829585.416306:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3020 00000100:00080000:2.0:1787829585.416312:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829585.416315:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829585.416318:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829585.416330:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f480 x1874672085863808/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829585.416332:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085863168:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1787829585.416358:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829585.416362:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829585.416364:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829585.416370:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3020 00000100:00080000:2.0:1787829585.416375:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829585.416378:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829585.416383:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:2.0:1787829585.416393:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e1c0 x1874672085863936/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829585.416400:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787829585.416405:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085863296:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1787829585.416406:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829585.416409:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829585.416418:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3020 00000100:00080000:2.0:1787829585.416421:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829585.416424:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829585.416429:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:2.0:1787829585.416439:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498d680 x1874672085864064/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829585.416462:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085863424:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829585.416494:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085863552:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829585.416516:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085863680:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829585.416539:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085863808:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829585.416559:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085863936:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829585.416578:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085864064:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829585.543919:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829585.543955:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829585.543960:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829585.544005:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829586.567910:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829586.567929:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829586.567931:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787829586.567992:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829587.591917:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829587.591955:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829587.591956:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829587.591959:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1787829588.488072:0:2397:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.141:988 00000400:00000100:2.0:1787829588.488121:0:2397:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.141@tcp at host 192.168.203.141 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1787829588.488150:0:2397:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:2.0:1787829588.488205:0:2397:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000800:00000100:2.0:1787829588.488220:0:2397:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:2.0:1787829588.488231:0:2397:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:2.0:1787829588.488243:0:2397:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:1.0:1787829588.488254:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829585/real 1787829588] req@ffff9e35c498d680 x1874672085864064/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829610 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:2.0:1787829588.488254:0:2397:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:2.0:1787829588.488268:0:2397:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:2.0:1787829588.488283:0:2397:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:1.0:1787829588.488295:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498d680 x1874672085864064/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829610 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:2.0:1787829588.488298:0:2397:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00080000:1.0:1787829588.488320:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829588.488326:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829588.488330:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085864064:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829588.488338:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829585/real 1787829588] req@ffff9e35c498e580 x1874672085863168/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829610 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829588.488351:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e580 x1874672085863168/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829610 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829588.488359:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829588.488363:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829588.488366:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085863168:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829588.488372:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829585/real 1787829588] req@ffff9e35c498fc00 x1874672085863296/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829610 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829588.488383:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498fc00 x1874672085863296/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829610 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829588.488390:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829588.488394:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829588.488397:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085863296:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829588.488404:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829585/real 1787829588] req@ffff9e35c498cb40 x1874672085863424/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829610 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829588.488417:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498cb40 x1874672085863424/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829610 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829588.488426:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829588.488430:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829588.488433:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085863424:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829588.488439:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829585/real 1787829588] req@ffff9e35c498c000 x1874672085863552/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829610 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829588.488451:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c000 x1874672085863552/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829610 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829588.488460:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829588.488463:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829588.488466:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085863552:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829588.488472:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829585/real 1787829588] req@ffff9e35c498f0c0 x1874672085863680/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829610 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829588.488482:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f0c0 x1874672085863680/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829610 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829588.488489:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829588.488492:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829588.488495:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085863680:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829588.488501:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829585/real 1787829588] req@ffff9e35c498e940 x1874672085863040/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829610 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829588.488511:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e940 x1874672085863040/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829610 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829588.488519:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829588.488522:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829588.488525:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085863040:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1787829588.488563:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829585/real 1787829588] req@ffff9e35c498f480 x1874672085863808/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829610 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829588.488576:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f480 x1874672085863808/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829610 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829588.488585:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829588.488588:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829588.488592:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085863808:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829588.488599:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829585/real 1787829588] req@ffff9e35c498e1c0 x1874672085863936/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829610 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829588.488611:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e1c0 x1874672085863936/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829610 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829588.488620:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829588.488624:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829588.488628:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085863936:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829588.615924:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829588.615924:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829588.615957:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787829588.615974:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829589.639871:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829589.639886:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829589.639908:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829589.639911:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829590.535877:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829590.535885:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829590.535888:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829590.535897:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3025 00000100:00080000:2.0:1787829590.535907:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829590.535913:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787829590.535944:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c780 x1874672085864192/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829590.535966:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829590.535969:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829590.535970:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829590.535973:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3025 00000100:00080000:2.0:1787829590.535976:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829590.535978:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829590.535987:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829590.535996:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498de00 x1874672085864320/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829590.536004:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085864192:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1787829590.536006:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829590.536013:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829590.536014:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829590.536018:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3025 00000100:00080000:2.0:1787829590.536022:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829590.536026:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829590.536031:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829590.536044:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498ed00 x1874672085864448/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829590.536055:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829590.536058:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829590.536060:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829590.536064:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3025 00000100:00080000:2.0:1787829590.536068:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829590.536071:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:1.0:1787829590.536077:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:2.0:1787829590.536079:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:2.0:1787829590.536095:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f840 x1874672085864576/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829590.536105:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829590.536109:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829590.536111:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829590.536132:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3025 00000100:00080000:2.0:1787829590.536137:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1787829590.536138:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829590/real 1787829590] req@ffff9e35c498c780 x1874672085864192/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829615 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829590.536145:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829590.536156:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:1.0:1787829590.536167:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c780 x1874672085864192/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829615 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829590.536174:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cf00 x1874672085864704/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829590.536181:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829590.536189:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:2.0:1787829590.536191:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829590.536198:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829590.536200:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787829590.536207:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085864192:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1787829590.536213:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3025 00000100:00080000:2.0:1787829590.536220:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829590.536224:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829590.536231:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829590.536240:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085864320:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829590.536247:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c3c0 x1874672085864832/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829590.536261:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829590.536264:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829590.536266:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1787829590.536273:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1787829590.536273:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3025 00000100:00080000:2.0:1787829590.536282:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829590.536287:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829590.536293:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829590.536295:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829590/real 1787829590] req@ffff9e35c498de00 x1874672085864320/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829615 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829590.536309:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c780 x1874672085864960/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829590.536314:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498de00 x1874672085864320/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829615 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829590.536335:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829590.536338:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829590.536341:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829590.536345:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:2.0:1787829590.536346:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787829590.536353:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085864320:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1787829590.536357:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3025 00000100:00080000:2.0:1787829590.536365:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1787829590.536366:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085864448:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1787829590.536373:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829590.536379:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:2.0:1787829590.536394:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e1c0 x1874672085865088/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787829590.536399:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1787829590.536411:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787829590.536416:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829590/real 1787829590] req@ffff9e35c498ed00 x1874672085864448/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829615 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829590.536417:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829590.536421:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829590.536431:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3025 00000100:00080000:2.0:1787829590.536438:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829590.536446:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829590.536453:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:1.0:1787829590.536462:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498ed00 x1874672085864448/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829615 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829590.536469:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f480 x1874672085865216/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829590.536482:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829590.536490:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829590.536495:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085864448:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829590.536504:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085864576:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829590.536527:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829590.536540:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829590/real 1787829590] req@ffff9e35c498f840 x1874672085864576/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829615 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829590.536552:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f840 x1874672085864576/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829615 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829590.536560:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829590.536563:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829590.536566:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085864576:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829590.536572:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085864704:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829590.536590:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829590.536600:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829590/real 1787829590] req@ffff9e35c498cf00 x1874672085864704/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829615 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829590.536611:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498cf00 x1874672085864704/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829615 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829590.536624:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829590.536627:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829590.536630:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085864704:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829590.536663:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085864832:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787829590.536835:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829590.536851:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829590/real 1787829590] req@ffff9e35c498c3c0 x1874672085864832/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829615 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829590.536864:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c3c0 x1874672085864832/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829615 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829590.536872:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829590.536876:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829590.536879:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085864832:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829590.536886:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085864960:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787829590.536909:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829590.536920:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829590/real 1787829590] req@ffff9e35c498c780 x1874672085864960/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829615 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829590.536931:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c780 x1874672085864960/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829615 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829590.536939:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829590.536942:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829590.536945:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085864960:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829590.536952:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085865088:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829590.536969:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829590.536980:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829590/real 1787829590] req@ffff9e35c498e1c0 x1874672085865088/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829615 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829590.536991:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e1c0 x1874672085865088/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829615 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829590.537001:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829590.537004:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829590.537007:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085865088:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829590.537013:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085865216:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829590.537030:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829590.537041:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829590/real 1787829590] req@ffff9e35c498f480 x1874672085865216/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829615 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829590.537055:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f480 x1874672085865216/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829615 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829590.537064:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829590.537068:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829590.537071:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085865216:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829590.663913:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829590.663916:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829590.663922:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829590.663966:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829591.687876:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829591.687888:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787829591.687905:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829591.687915:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829592.711901:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829592.711901:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829592.711934:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829592.711950:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829593.735851:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829593.735857:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829593.735881:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829593.735939:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829594.759944:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829594.759949:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829594.759972:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787829594.761313:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829595.655918:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829595.655927:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829595.655931:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829595.655941:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3031 00000100:00080000:2.0:1787829595.655953:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829595.655960:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787829595.656027:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e940 x1874672085865344/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829595.656054:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829595.656058:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829595.656060:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829595.656064:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3031 00000100:00080000:2.0:1787829595.656068:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829595.656072:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829595.656094:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829595.656107:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f0c0 x1874672085865472/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829595.656117:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829595.656121:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829595.656122:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829595.656127:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3031 00000100:00080000:2.0:1787829595.656137:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829595.656140:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829595.656145:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829595.656148:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085865344:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1787829595.656157:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c000 x1874672085865600/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829595.656168:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829595.656171:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829595.656173:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829595.656177:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3031 00000100:00080000:2.0:1787829595.656181:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829595.656185:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829595.656189:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:2.0:1787829595.656201:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cb40 x1874672085865728/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829595.656210:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829595.656214:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829595.656216:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829595.656220:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3031 00000100:00080000:2.0:1787829595.656224:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829595.656227:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829595.656233:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000800:00000100:1.0:1787829595.656239:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829595.656245:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498fc00 x1874672085865856/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829595.656256:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829595.656260:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829595.656262:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829595.656267:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3031 00000100:00080000:2.0:1787829595.656270:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829595.656274:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829595.656278:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829595.656289:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e580 x1874672085865984/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829595.656299:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829595/real 1787829595] req@ffff9e35c498e940 x1874672085865344/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829620 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829595.656300:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829595.656308:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829595.656314:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829595.656320:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3031 00000100:00100000:1.0:1787829595.656321:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e940 x1874672085865344/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829620 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829595.656327:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829595.656340:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829595.656343:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829595.656347:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085865344:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1787829595.656355:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085865472:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787829595.656403:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1787829595.656404:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829595.656412:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829595.656417:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829595/real 1787829595] req@ffff9e35c498f0c0 x1874672085865472/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829620 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829595.656427:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498d680 x1874672085866112/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829595.656453:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829595.656457:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829595.656458:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787829595.656462:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f0c0 x1874672085865472/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829620 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829595.656464:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3031 00000100:00080000:2.0:1787829595.656472:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829595.656475:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829595.656479:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:1.0:1787829595.656483:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000008:00100000:2.0:1787829595.656487:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:1.0:1787829595.656489:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085865472:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829595.656504:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f480 x1874672085866240/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829595.656514:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829595.656517:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829595.656519:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787829595.656524:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085865600:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1787829595.656534:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3031 00000100:00080000:2.0:1787829595.656539:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829595.656544:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:1.0:1787829595.656549:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:2.0:1787829595.656551:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:2.0:1787829595.656565:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e1c0 x1874672085866368/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829595.656570:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829595/real 1787829595] req@ffff9e35c498c000 x1874672085865600/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829620 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829595.656588:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c000 x1874672085865600/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829620 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829595.656593:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829595.656596:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829595.656598:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085865600:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829595.656602:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085865728:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829595.656617:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829595.656623:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829595/real 1787829595] req@ffff9e35c498cb40 x1874672085865728/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829620 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829595.656629:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498cb40 x1874672085865728/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829620 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829595.656634:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829595.656636:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829595.656637:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085865728:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829595.656640:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085865856:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829595.656649:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829595.656655:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829595/real 1787829595] req@ffff9e35c498fc00 x1874672085865856/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829620 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829595.656661:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498fc00 x1874672085865856/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829620 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829595.656666:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829595.656667:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829595.656669:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085865856:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829595.656672:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085865984:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787829595.656681:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829595.656686:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829595/real 1787829595] req@ffff9e35c498e580 x1874672085865984/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829620 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829595.656692:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e580 x1874672085865984/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829620 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829595.656696:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829595.656698:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829595.656699:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085865984:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829595.656702:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085866112:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787829595.656714:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829595.656723:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829595/real 1787829595] req@ffff9e35c498d680 x1874672085866112/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829620 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829595.656734:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498d680 x1874672085866112/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829620 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829595.656748:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829595.656751:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829595.656754:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085866112:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829595.656758:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085866240:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829595.656772:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829595.656778:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829595/real 1787829595] req@ffff9e35c498f480 x1874672085866240/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829620 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829595.656784:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f480 x1874672085866240/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829620 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829595.656789:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829595.656791:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829595.656793:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085866240:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829595.656836:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085866368:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829595.656853:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829595.656860:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829595/real 1787829595] req@ffff9e35c498e1c0 x1874672085866368/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829620 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829595.656869:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e1c0 x1874672085866368/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829620 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829595.656877:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829595.656880:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829595.656883:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085866368:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829595.783915:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829595.783918:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829595.783951:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829595.784054:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829596.807891:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829596.807894:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829596.807909:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829596.807923:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829597.831917:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829597.831925:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829597.831947:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787829597.831959:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829598.855932:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829598.855932:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829598.855959:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829598.855984:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829599.879897:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829599.879897:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829599.879927:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829599.879947:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829600.775868:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829600.775876:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829600.775879:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829600.775888:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3036 00000100:00080000:2.0:1787829600.775898:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829600.775905:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787829600.775939:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c780 x1874672085866496/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829600.775958:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829600.775961:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829600.775962:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829600.775965:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3036 00000100:00080000:2.0:1787829600.775968:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829600.775971:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829600.775979:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829600.775988:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c3c0 x1874672085866624/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829600.775995:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829600.775998:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829600.776000:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787829600.776003:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085866496:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1787829600.776005:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3036 00000100:00080000:2.0:1787829600.776007:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829600.776009:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829600.776012:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829600.776020:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cf00 x1874672085866752/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829600.776027:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829600.776028:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829600.776029:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829600.776032:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3036 00000100:00080000:2.0:1787829600.776033:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829600.776036:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829600.776038:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:2.0:1787829600.776063:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f840 x1874672085866880/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829600.776070:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829600.776072:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829600.776073:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829600.776076:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3036 00000100:00080000:2.0:1787829600.776079:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829600.776081:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829600.776085:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000800:00000100:1.0:1787829600.776090:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829600.776095:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498ed00 x1874672085867008/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829600.776102:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829600.776105:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829600.776106:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829600.776109:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3036 00000100:00080000:2.0:1787829600.776111:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829600.776112:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829600.776115:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829600.776121:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498de00 x1874672085867136/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829600.776127:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787829600.776131:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829600/real 1787829600] req@ffff9e35c498c780 x1874672085866496/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829625 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829600.776131:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829600.776134:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829600.776141:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3036 00000100:00080000:2.0:1787829600.776145:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829600.776150:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829600.776153:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829600.776159:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c780 x1874672085866496/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829625 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829600.776161:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e1c0 x1874672085867264/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829600.776185:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829600.776187:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829600.776188:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829600.776192:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829600.776196:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:2.0:1787829600.776197:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3036 00000100:00080000:2.0:1787829600.776203:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1787829600.776204:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085866496:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1787829600.776210:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829600.776215:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:2.0:1787829600.776226:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f480 x1874672085867392/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829600.776231:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085866624:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1787829600.776246:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829600.776250:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829600.776252:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1787829600.776260:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1787829600.776260:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3036 00000100:00080000:2.0:1787829600.776267:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829600.776271:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829600.776275:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:1.0:1787829600.776276:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829600/real 1787829600] req@ffff9e35c498c3c0 x1874672085866624/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829625 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829600.776287:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498d680 x1874672085867520/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829600.776294:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c3c0 x1874672085866624/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829625 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829600.776304:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829600.776308:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829600.776312:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085866624:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829600.776319:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085866752:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787829600.776335:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829600.776344:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829600/real 1787829600] req@ffff9e35c498cf00 x1874672085866752/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829625 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829600.776354:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498cf00 x1874672085866752/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829625 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829600.776362:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829600.776365:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829600.776368:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085866752:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829600.776373:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085866880:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829600.776387:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829600.776395:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829600/real 1787829600] req@ffff9e35c498f840 x1874672085866880/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829625 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829600.776405:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f840 x1874672085866880/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829625 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829600.776412:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829600.776415:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829600.776418:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085866880:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829600.776423:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085867008:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829600.776436:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829600.776453:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829600/real 1787829600] req@ffff9e35c498ed00 x1874672085867008/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829625 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829600.776464:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498ed00 x1874672085867008/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829625 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829600.776472:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829600.776474:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829600.776478:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085867008:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829600.776482:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085867136:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787829600.776521:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829600.776531:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829600/real 1787829600] req@ffff9e35c498de00 x1874672085867136/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829625 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829600.776541:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498de00 x1874672085867136/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829625 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829600.776549:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829600.776558:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829600.776561:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085867136:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829600.776566:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085867264:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787829600.776581:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829600.776590:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829600/real 1787829600] req@ffff9e35c498e1c0 x1874672085867264/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829625 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829600.776600:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e1c0 x1874672085867264/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829625 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829600.776614:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829600.776617:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829600.776620:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085867264:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829600.776641:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085867392:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829600.776656:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829600.776665:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829600/real 1787829600] req@ffff9e35c498f480 x1874672085867392/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829625 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829600.776676:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f480 x1874672085867392/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829625 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829600.776684:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829600.776686:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829600.776689:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085867392:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829600.776695:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085867520:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829600.776707:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829600.776716:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829600/real 1787829600] req@ffff9e35c498d680 x1874672085867520/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829625 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829600.776726:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498d680 x1874672085867520/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829625 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829600.776734:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829600.776737:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829600.776741:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085867520:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829600.903887:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829600.903888:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829600.903888:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829600.903946:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829601.927900:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829601.927900:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829601.927911:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787829601.927945:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829602.951871:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829602.951872:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829602.951874:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829602.951922:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829603.975887:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829603.975887:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829603.975907:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829603.975922:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829604.999910:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829604.999918:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787829604.999953:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829605.000094:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00080000:2.0:1787829605.895877:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829605.895888:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829605.895893:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829605.895904:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3041 00000100:00080000:2.0:1787829605.895917:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829605.895927:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787829605.895977:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e580 x1874672085867648/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829605.896022:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829605.896026:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829605.896028:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829605.896034:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3041 00000100:00080000:2.0:1787829605.896041:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829605.896047:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829605.896071:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829605.896077:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085867648:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1787829605.896096:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498fc00 x1874672085867776/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829605.896113:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829605.896117:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829605.896120:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829605.896125:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3041 00000100:00080000:2.0:1787829605.896130:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829605.896134:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829605.896141:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829605.896158:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cb40 x1874672085867904/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829605.896174:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829605.896179:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829605.896181:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829605.896186:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3041 00000100:00080000:2.0:1787829605.896191:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829605.896196:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829605.896203:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000800:00000100:0.0:1787829605.896211:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829605.896225:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c000 x1874672085868032/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829605.896239:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829605.896242:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829605.896244:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829605.896248:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3041 00000100:00080000:2.0:1787829605.896253:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829605.896261:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787829605.896263:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829605/real 1787829605] req@ffff9e35c498e580 x1874672085867648/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829630 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1787829605.896267:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:2.0:1787829605.896284:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f0c0 x1874672085868160/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829605.896292:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787829605.896296:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e580 x1874672085867648/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829630 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829605.896298:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829605.896301:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829605.896310:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3041 00000100:00080000:2.0:1787829605.896317:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829605.896328:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829605.896332:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1787829605.896333:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829605.896340:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829605.896345:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e940 x1874672085868288/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829605.896350:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085867648:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1787829605.896376:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829605.896379:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829605.896381:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829605.896385:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3041 00000100:00080000:2.0:1787829605.896387:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829605.896392:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787829605.896398:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085867776:192.168.203.141@tcp:38:kworker.0 00000008:00100000:2.0:1787829605.896406:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829605.896430:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e580 x1874672085868416/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829605.896431:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1787829605.896448:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787829605.896456:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829605/real 1787829605] req@ffff9e35c498fc00 x1874672085867776/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829630 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829605.896457:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829605.896462:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829605.896474:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3041 00000100:00080000:2.0:1787829605.896480:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829605.896486:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829605.896493:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:0.0:1787829605.896499:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498fc00 x1874672085867776/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829630 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829605.896509:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498d680 x1874672085868544/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829605.896516:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829605.896535:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:2.0:1787829605.896537:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829605.896544:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829605.896546:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787829605.896552:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085867776:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1787829605.896557:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3041 00000100:00080000:2.0:1787829605.896564:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787829605.896568:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085867904:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1787829605.896574:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829605.896583:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000800:00000100:0.0:1787829605.896597:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829605.896602:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f480 x1874672085868672/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829605.896611:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829605/real 1787829605] req@ffff9e35c498cb40 x1874672085867904/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829630 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829605.896635:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498cb40 x1874672085867904/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829630 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829605.896645:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829605.896649:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829605.896652:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085867904:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829605.896657:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085868032:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829605.896670:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829605.896676:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829605/real 1787829605] req@ffff9e35c498c000 x1874672085868032/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829630 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829605.896683:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c000 x1874672085868032/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829630 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829605.896689:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829605.896691:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829605.896692:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085868032:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829605.896697:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085868160:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829605.896706:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829605.896712:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829605/real 1787829605] req@ffff9e35c498f0c0 x1874672085868160/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829630 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829605.896719:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f0c0 x1874672085868160/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829630 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829605.896729:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829605.896731:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829605.896733:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085868160:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829605.896736:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085868288:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829605.896746:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829605.896752:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829605/real 1787829605] req@ffff9e35c498e940 x1874672085868288/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829630 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829605.896759:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e940 x1874672085868288/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829630 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829605.896764:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829605.896766:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829605.896768:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085868288:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829605.896786:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085868416:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829605.896843:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829605.896857:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829605/real 1787829605] req@ffff9e35c498e580 x1874672085868416/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829630 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829605.896870:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e580 x1874672085868416/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829630 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829605.896878:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829605.896881:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829605.896884:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085868416:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829605.896889:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085868544:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829605.896905:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829605.896912:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829605/real 1787829605] req@ffff9e35c498d680 x1874672085868544/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829630 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829605.896918:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498d680 x1874672085868544/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829630 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829605.896924:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829605.896926:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829605.896928:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085868544:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829605.896931:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085868672:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829605.896940:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829605.896946:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829605/real 1787829605] req@ffff9e35c498f480 x1874672085868672/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829630 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829605.896953:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f480 x1874672085868672/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829630 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829605.896958:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829605.896960:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829605.896961:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085868672:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829606.023948:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829606.023948:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829606.023958:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829606.023996:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829607.047893:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829607.047893:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:3.0:1787829607.047893:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829607.047985:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829608.071906:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829608.071934:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829608.071935:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787829608.071948:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829609.095892:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829609.095899:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829609.095939:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829609.096065:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829610.119913:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829610.119913:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829610.119924:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787829610.119963:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829611.015905:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829611.015916:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829611.015921:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829611.015932:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3046 00000100:00080000:2.0:1787829611.015943:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829611.015951:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787829611.016005:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e1c0 x1874672085868800/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829611.016036:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829611.016040:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829611.016042:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829611.016047:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3046 00000100:00080000:2.0:1787829611.016051:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829611.016056:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829611.016068:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829611.016083:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498de00 x1874672085868928/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829611.016088:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085868800:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1787829611.016104:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829611.016108:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829611.016111:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829611.016116:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3046 00000100:00080000:2.0:1787829611.016120:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829611.016125:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829611.016130:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829611.016156:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498ed00 x1874672085869056/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829611.016160:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1787829611.016173:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829611.016177:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829611.016179:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829611.016184:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3046 00000100:00080000:2.0:1787829611.016188:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829611.016191:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829611.016196:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:2.0:1787829611.016207:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f840 x1874672085869184/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829611.016216:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829611.016220:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829611.016222:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829611.016227:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3046 00000100:00080000:2.0:1787829611.016231:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829611.016235:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829611.016240:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:2.0:1787829611.016254:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cf00 x1874672085869312/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829611.016263:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829611/real 1787829611] req@ffff9e35c498e1c0 x1874672085868800/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829636 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829611.016268:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829611.016277:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829611.016280:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829611.016287:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3046 00000100:00080000:2.0:1787829611.016292:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787829611.016295:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e1c0 x1874672085868800/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829636 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829611.016300:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829611.016309:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1787829611.016314:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829611.016319:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829611.016326:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085868800:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1787829611.016332:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c3c0 x1874672085869440/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829611.016345:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829611.016348:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829611.016351:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787829611.016356:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085868928:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1787829611.016358:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3046 00000100:00080000:2.0:1787829611.016365:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829611.016371:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829611.016377:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829611.016390:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c780 x1874672085869568/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829611.016395:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1787829611.016409:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829611.016413:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829611.016418:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787829611.016419:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829611/real 1787829611] req@ffff9e35c498de00 x1874672085868928/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829636 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829611.016428:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3046 00000100:00080000:2.0:1787829611.016437:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829611.016443:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787829611.016447:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498de00 x1874672085868928/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829636 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1787829611.016451:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00080000:0.0:1787829611.016464:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829611.016470:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829611.016470:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e1c0 x1874672085869696/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829611.016477:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085868928:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1787829611.016492:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787829611.016493:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085869056:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1787829611.016499:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829611.016504:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829611.016509:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3046 00000100:00080000:2.0:1787829611.016513:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829611.016518:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829611.016524:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000800:00000100:0.0:1787829611.016536:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829611.016540:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f480 x1874672085869824/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829611.016555:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829611/real 1787829611] req@ffff9e35c498ed00 x1874672085869056/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829636 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829611.016577:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498ed00 x1874672085869056/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829636 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829611.016587:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829611.016590:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829611.016594:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085869056:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829611.016600:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085869184:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829611.016620:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829611.016630:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829611/real 1787829611] req@ffff9e35c498f840 x1874672085869184/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829636 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829611.016638:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f840 x1874672085869184/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829636 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829611.016644:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829611.016646:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829611.016649:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085869184:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829611.016652:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085869312:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829611.016664:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829611.016670:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829611/real 1787829611] req@ffff9e35c498cf00 x1874672085869312/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829636 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829611.016677:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498cf00 x1874672085869312/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829636 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829611.016687:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829611.016689:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829611.016691:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085869312:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829611.016694:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085869440:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829611.016704:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829611.016710:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829611/real 1787829611] req@ffff9e35c498c3c0 x1874672085869440/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829636 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829611.016722:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c3c0 x1874672085869440/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829636 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829611.016731:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829611.016735:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829611.016738:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085869440:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829611.016763:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085869568:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829611.016786:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829611.016834:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829611/real 1787829611] req@ffff9e35c498c780 x1874672085869568/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829636 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829611.016850:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c780 x1874672085869568/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829636 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829611.016859:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829611.016862:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829611.016866:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085869568:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829611.016873:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085869696:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829611.016894:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829611.016907:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829611/real 1787829611] req@ffff9e35c498e1c0 x1874672085869696/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829636 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829611.016920:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e1c0 x1874672085869696/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829636 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829611.016929:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829611.016932:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829611.016935:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085869696:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829611.016941:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085869824:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829611.016958:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829611.016988:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829611/real 1787829611] req@ffff9e35c498f480 x1874672085869824/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829636 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829611.017005:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f480 x1874672085869824/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829636 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829611.017010:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829611.017012:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829611.017015:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085869824:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829611.143894:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829611.143931:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829611.144027:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787829611.144073:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829612.167944:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829612.167951:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829612.167999:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787829612.168016:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829613.191910:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829613.191932:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829613.191981:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829613.191993:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829614.215884:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829614.215891:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829614.215960:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787829614.215985:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829615.239934:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829615.239937:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829615.239938:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787829615.240013:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829616.135962:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829616.135973:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787829616.135977:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829616.135999:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3051 00000100:00080000:0.0:1787829616.136011:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829616.136019:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787829616.136055:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7192c0 x1874672085869952/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829616.136096:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829616.136101:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829616.136103:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829616.136108:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3051 00000100:00080000:0.0:1787829616.136112:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829616.136117:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829616.136129:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829616.136136:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085869952:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1787829616.136160:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a1c0 x1874672085870080/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829616.136174:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829616.136178:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787829616.136180:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829616.136185:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3051 00000100:00080000:0.0:1787829616.136190:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1787829616.136194:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787829616.136196:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829616.136205:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829616.136219:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718b40 x1874672085870208/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829616.136231:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787829616.136232:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829616/real 1787829616] req@ffff9e35da7192c0 x1874672085869952/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829641 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829616.136239:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829616.136244:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829616.136253:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3051 00000100:00080000:0.0:1787829616.136259:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1787829616.136263:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7192c0 x1874672085869952/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829641 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829616.136269:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829616.136279:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00080000:1.0:1787829616.136284:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829616.136292:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829616.136297:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718000 x1874672085870336/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829616.136300:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085869952:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1787829616.136317:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829616.136321:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829616.136323:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787829616.136328:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085870080:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787829616.136331:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3051 00000100:00080000:0.0:1787829616.136339:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829616.136344:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829616.136350:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000800:00000100:1.0:1787829616.136354:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829616.136371:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a580 x1874672085870464/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829616.136371:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829616/real 1787829616] req@ffff9e35da71a1c0 x1874672085870080/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829641 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829616.136406:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a1c0 x1874672085870080/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829641 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829616.136408:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829616.136414:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829616.136416:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829616.136418:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085870080:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787829616.136428:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:1.0:1787829616.136431:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085870208:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787829616.136433:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829616.136443:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3051 00000100:00080000:0.0:1787829616.136450:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829616.136455:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829616.136462:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1787829616.136463:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829616.136491:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da719a40 x1874672085870592/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829616.136491:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829616/real 1787829616] req@ffff9e35da718b40 x1874672085870208/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829641 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829616.136529:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718b40 x1874672085870208/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829641 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829616.136530:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829616.136541:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787829616.136546:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829616.136547:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829616.136555:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:0.0:1787829616.136558:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3051 00000100:00100000:1.0:1787829616.136561:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085870208:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787829616.136567:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1787829616.136568:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085870336:192.168.203.141@tcp:8:kworker.0 00000100:00080000:0.0:1787829616.136577:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829616.136585:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1787829616.136591:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829616.136602:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7183c0 x1874672085870720/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829616.136603:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829616/real 1787829616] req@ffff9e35da718000 x1874672085870336/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829641 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829616.136638:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718000 x1874672085870336/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829641 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829616.136640:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829616.136652:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829616.136654:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829616.136656:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829616.136659:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829616.136667:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085870336:192.168.203.141@tcp:8:kworker.0 00000100:00080000:0.0:1787829616.136668:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3051 00000100:00080000:0.0:1787829616.136677:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829616.136682:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829616.136688:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:1.0:1787829616.136689:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085870464:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829616.136706:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a940 x1874672085870848/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787829616.136718:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787829616.136720:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829616.136726:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829616.136728:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787829616.136734:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829616/real 1787829616] req@ffff9e35da71a580 x1874672085870464/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829641 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829616.136735:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3051 00000100:00080000:0.0:1787829616.136744:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1787829616.136750:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672085870464/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829641 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829616.136753:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829616.136762:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00080000:1.0:1787829616.136766:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829616.136773:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829616.136779:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718f00 x1874672085870976/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829616.136780:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085870464:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829616.136793:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085870592:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787829616.136856:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829616.136867:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829616/real 1787829616] req@ffff9e35da719a40 x1874672085870592/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829641 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829616.136881:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672085870592/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829641 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829616.136886:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829616.136888:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829616.136891:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085870592:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829616.136894:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085870720:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787829616.136904:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829616.136910:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829616/real 1787829616] req@ffff9e35da7183c0 x1874672085870720/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829641 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829616.136917:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672085870720/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829641 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829616.136922:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829616.136924:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829616.136926:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085870720:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829616.136950:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085870848:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829616.136965:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829616.136973:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829616/real 1787829616] req@ffff9e35da71a940 x1874672085870848/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829641 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829616.136980:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672085870848/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829641 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829616.136990:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829616.136992:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829616.136994:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085870848:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829616.136998:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085870976:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829616.137007:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829616.137012:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829616/real 1787829616] req@ffff9e35da718f00 x1874672085870976/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829641 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829616.137019:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672085870976/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829641 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829616.137025:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829616.137027:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829616.137028:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085870976:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829616.263904:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829616.263925:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829616.263958:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829616.263994:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829617.287921:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829617.287921:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829617.287924:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787829617.288011:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829618.311898:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829618.311898:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829618.311941:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787829618.311963:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829619.335887:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829619.335893:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787829619.335944:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829619.335950:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829620.359960:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829620.360001:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787829620.360034:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829620.360043:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00080000:2.0:1787829621.255967:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829621.255979:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829621.255984:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829621.255996:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3056 00000100:00080000:2.0:1787829621.256009:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829621.256018:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787829621.256060:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498d680 x1874672085871104/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829621.256088:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829621.256095:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829621.256096:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829621.256100:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3056 00000100:00080000:2.0:1787829621.256105:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829621.256111:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829621.256123:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829621.256141:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e580 x1874672085871232/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829621.256153:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085871104:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1787829621.256162:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829621.256168:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829621.256171:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829621.256177:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3056 00000100:00080000:2.0:1787829621.256181:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829621.256185:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829621.256190:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829621.256201:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e940 x1874672085871360/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829621.256210:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829621.256212:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829621.256214:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829621.256218:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3056 00000100:00080000:2.0:1787829621.256221:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829621.256224:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829621.256229:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:2.0:1787829621.256239:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f0c0 x1874672085871488/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787829621.256250:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1787829621.256250:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829621.256257:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829621.256259:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829621.256263:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3056 00000100:00080000:2.0:1787829621.256266:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829621.256270:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829621.256285:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:2.0:1787829621.256303:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c000 x1874672085871616/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829621.256308:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829621/real 1787829621] req@ffff9e35c498d680 x1874672085871104/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829646 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829621.256334:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829621.256343:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829621.256346:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829621.256355:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3056 00000100:00080000:2.0:1787829621.256362:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829621.256368:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829621.256379:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829621.256390:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498d680 x1874672085871104/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829646 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829621.256398:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cb40 x1874672085871744/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829621.256410:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829621.256418:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829621.256420:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:2.0:1787829621.256425:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829621.256428:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787829621.256432:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085871104:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1787829621.256435:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3056 00000100:00080000:2.0:1787829621.256443:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829621.256448:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829621.256453:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829621.256462:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498fc00 x1874672085871872/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829621.256471:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085871232:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1787829621.256472:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829621.256480:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829621.256484:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829621.256488:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3056 00000100:00080000:2.0:1787829621.256492:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829621.256495:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829621.256498:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:2.0:1787829621.256508:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498d680 x1874672085872000/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829621.256515:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829621.256518:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829621.256519:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1787829621.256525:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1787829621.256528:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3056 00000100:00080000:2.0:1787829621.256533:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829621.256536:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829621.256541:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:1.0:1787829621.256552:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829621/real 1787829621] req@ffff9e35c498e580 x1874672085871232/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829646 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829621.256553:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f480 x1874672085872128/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829621.256600:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e580 x1874672085871232/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829646 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829621.256611:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829621.256614:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829621.256616:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085871232:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829621.256622:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085871360:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787829621.256642:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829621.256668:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829621/real 1787829621] req@ffff9e35c498e940 x1874672085871360/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829646 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829621.256678:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e940 x1874672085871360/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829646 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829621.256690:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829621.256693:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829621.256696:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085871360:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829621.256700:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085871488:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829621.256715:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829621.256722:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829621/real 1787829621] req@ffff9e35c498f0c0 x1874672085871488/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829646 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829621.256731:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f0c0 x1874672085871488/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829646 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829621.256737:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829621.256740:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829621.256742:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085871488:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829621.256747:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085871616:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829621.256759:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829621.256783:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829621/real 1787829621] req@ffff9e35c498c000 x1874672085871616/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829646 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829621.256791:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c000 x1874672085871616/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829646 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829621.256797:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829621.256799:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829621.256802:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085871616:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829621.256805:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085871744:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787829621.256818:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829621.256824:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829621/real 1787829621] req@ffff9e35c498cb40 x1874672085871744/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829646 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829621.256853:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498cb40 x1874672085871744/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829646 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829621.256859:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829621.256862:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829621.256864:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085871744:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829621.256884:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085871872:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787829621.256915:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829621.256928:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829621/real 1787829621] req@ffff9e35c498fc00 x1874672085871872/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829646 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829621.256961:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498fc00 x1874672085871872/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829646 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829621.256968:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829621.256970:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829621.256973:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085871872:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829621.256977:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085872000:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829621.256994:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829621.257002:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829621/real 1787829621] req@ffff9e35c498d680 x1874672085872000/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829646 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829621.257011:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498d680 x1874672085872000/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829646 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829621.257017:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829621.257019:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829621.257021:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085872000:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829621.257025:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085872128:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829621.257036:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829621.257043:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829621/real 1787829621] req@ffff9e35c498f480 x1874672085872128/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829646 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829621.257052:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f480 x1874672085872128/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829646 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829621.257058:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829621.257060:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829621.257062:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085872128:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829621.383910:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829621.383915:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829621.383918:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:0.0:1787829621.383964:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829622.407865:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787829622.407911:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829622.407914:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829622.407929:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829623.431877:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829623.431877:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829623.431928:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787829623.431957:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829624.455873:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829624.455911:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829624.455929:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829624.455933:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829625.479913:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829625.479936:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787829625.479936:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829625.479958:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00080000:3.0:1787829626.375931:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829626.375941:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787829626.375946:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829626.375956:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3061 00000100:00080000:3.0:1787829626.375967:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829626.375975:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787829626.376033:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a580 x1874672085872256/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829626.376061:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829626.376065:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829626.376067:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829626.376072:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3061 00000100:00080000:3.0:1787829626.376076:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829626.376081:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829626.376091:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829626.376106:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085872256:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1787829626.376107:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499e00 x1874672085872384/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829626.376140:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829626.376144:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787829626.376147:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829626.376152:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3061 00000100:00080000:3.0:1787829626.376157:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829626.376163:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829626.376169:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829626.376184:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b0c0 x1874672085872512/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787829626.376189:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1787829626.376200:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829626.376203:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787829626.376205:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829626.376210:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3061 00000100:00080000:3.0:1787829626.376221:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829626.376225:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829626.376230:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:1.0:1787829626.376235:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829626/real 1787829626] req@ffff9e35c649a580 x1874672085872256/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829651 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829626.376244:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a940 x1874672085872640/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829626.376261:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829626.376263:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787829626.376265:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787829626.376270:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a580 x1874672085872256/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829651 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829626.376272:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3061 00000100:00080000:3.0:1787829626.376277:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829626.376283:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829626.376291:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00080000:1.0:1787829626.376294:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829626.376300:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787829626.376303:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499680 x1874672085872768/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829626.376332:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085872256:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1787829626.376347:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829626.376351:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829626.376353:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829626.376358:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3061 00000100:00100000:1.0:1787829626.376362:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085872384:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1787829626.376364:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829626.376371:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829626.376378:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1787829626.376386:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787829626.376395:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b840 x1874672085872896/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829626.376400:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829626/real 1787829626] req@ffff9e35c6499e00 x1874672085872384/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829651 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829626.376408:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829626.376412:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787829626.376414:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787829626.376415:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499e00 x1874672085872384/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829651 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829626.376419:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3061 00000100:00080000:3.0:1787829626.376426:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829626.376431:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829626.376432:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829626.376439:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00080000:1.0:1787829626.376440:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829626.376446:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085872384:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787829626.376454:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649bc00 x1874672085873024/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829626.376455:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085872512:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1787829626.376468:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829626.376471:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787829626.376473:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1787829626.376478:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1787829626.376478:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3061 00000100:00080000:3.0:1787829626.376485:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829626.376489:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787829626.376493:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829626/real 1787829626] req@ffff9e35c649b0c0 x1874672085872512/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829651 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1787829626.376495:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:3.0:1787829626.376510:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498b40 x1874672085873152/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829626.376517:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b0c0 x1874672085872512/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829651 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829626.376517:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829626.376519:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787829626.376521:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829626.376529:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3061 00000100:00080000:3.0:1787829626.376534:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829626.376540:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829626.376547:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00080000:1.0:1787829626.376552:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1787829626.376557:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a1c0 x1874672085873280/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829626.376558:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829626.376565:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085872512:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829626.376576:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085872640:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829626.376592:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829626.376601:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829626/real 1787829626] req@ffff9e35c649a940 x1874672085872640/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829651 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829626.376609:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a940 x1874672085872640/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829651 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829626.376614:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829626.376616:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829626.376618:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085872640:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829626.376621:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085872768:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829626.376631:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829626.376636:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829626/real 1787829626] req@ffff9e35c6499680 x1874672085872768/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829651 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829626.376643:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499680 x1874672085872768/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829651 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829626.376648:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829626.376650:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829626.376652:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085872768:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829626.376666:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085872896:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787829626.376677:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829626.376683:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829626/real 1787829626] req@ffff9e35c649b840 x1874672085872896/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829651 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829626.376691:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b840 x1874672085872896/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829651 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829626.376699:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829626.376702:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829626.376706:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085872896:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829626.376711:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085873024:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787829626.376729:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829626.376741:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829626/real 1787829626] req@ffff9e35c649bc00 x1874672085873024/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829651 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829626.376753:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649bc00 x1874672085873024/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829651 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829626.376759:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829626.376761:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829626.376763:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085873024:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829626.376767:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085873152:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829626.376781:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829626.376788:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829626/real 1787829626] req@ffff9e35c6498b40 x1874672085873152/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829651 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829626.376795:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498b40 x1874672085873152/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829651 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829626.376824:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829626.376826:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829626.376828:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085873152:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829626.376832:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085873280:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829626.376844:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829626.376851:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829626/real 1787829626] req@ffff9e35c649a1c0 x1874672085873280/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829651 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829626.376858:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a1c0 x1874672085873280/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829651 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829626.376882:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829626.376885:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829626.376888:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085873280:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829626.503901:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829626.503905:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829626.503916:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787829626.503954:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829627.527876:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829627.527892:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829627.527945:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829627.527974:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829628.551858:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829628.551896:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787829628.551935:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829628.551947:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829629.575912:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787829629.575949:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829629.576028:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829629.576071:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829630.599884:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829630.599894:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829630.599925:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829630.599926:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00080000:1.0:1787829631.495888:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829631.495896:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829631.495900:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829631.495908:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3066 00000100:00080000:1.0:1787829631.495918:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829631.495924:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787829631.495957:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7840 x1874672085873408/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829631.495983:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829631.495987:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829631.495990:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829631.495994:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3066 00000100:00080000:1.0:1787829631.495998:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829631.496003:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829631.496014:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829631.496030:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f43c0 x1874672085873536/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829631.496034:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085873408:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787829631.496043:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829631.496049:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829631.496051:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829631.496054:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3066 00000100:00080000:1.0:1787829631.496057:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829631.496060:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829631.496063:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829631.496073:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5a40 x1874672085873664/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829631.496080:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829631.496083:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829631.496084:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829631.496086:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3066 00000100:00080000:1.0:1787829631.496097:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829631.496099:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829631.496103:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000800:00000100:0.0:1787829631.496108:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829631.496113:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5e00 x1874672085873792/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829631.496121:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829631.496123:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829631.496124:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829631.496127:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3066 00000100:00080000:1.0:1787829631.496130:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829631.496132:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829631.496134:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:1.0:1787829631.496142:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4b40 x1874672085873920/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829631.496149:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787829631.496154:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829631/real 1787829631] req@ffff9e35da6f7840 x1874672085873408/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829656 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829631.496158:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829631.496162:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829631.496171:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3066 00000100:00080000:1.0:1787829631.496178:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787829631.496185:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7840 x1874672085873408/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829656 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829631.496185:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829631.496196:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1787829631.496206:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829631.496212:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829631.496216:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f70c0 x1874672085874048/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829631.496220:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085873408:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787829631.496234:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829631.496237:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829631.496238:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829631.496243:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3066 00000100:00080000:1.0:1787829631.496246:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829631.496249:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829631.496253:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829631.496262:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085873536:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829631.496265:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7840 x1874672085874176/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829631.496288:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829631.496292:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829631.496294:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1787829631.496302:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787829631.496303:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3066 00000100:00080000:1.0:1787829631.496311:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829631.496316:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787829631.496322:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829631/real 1787829631] req@ffff9e35da6f43c0 x1874672085873536/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829656 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1787829631.496323:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:1.0:1787829631.496339:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f61c0 x1874672085874304/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829631.496344:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f43c0 x1874672085873536/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829656 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829631.496377:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829631.496378:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829631.496384:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829631.496386:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:1.0:1787829631.496387:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787829631.496395:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085873536:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787829631.496395:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3066 00000100:00080000:1.0:1787829631.496401:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829631.496406:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787829631.496411:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085873664:192.168.203.141@tcp:38:kworker.0 00000008:00100000:1.0:1787829631.496412:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:1.0:1787829631.496424:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4f00 x1874672085874432/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829631.496440:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829631.496470:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829631/real 1787829631] req@ffff9e35da6f5a40 x1874672085873664/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829656 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829631.496502:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5a40 x1874672085873664/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829656 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829631.496517:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829631.496522:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829631.496527:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085873664:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829631.496552:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085873792:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829631.496581:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829631.496597:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829631/real 1787829631] req@ffff9e35da6f5e00 x1874672085873792/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829656 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829631.496627:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5e00 x1874672085873792/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829656 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829631.496636:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829631.496639:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829631.496643:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085873792:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829631.496649:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085873920:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829631.496665:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829631.496674:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829631/real 1787829631] req@ffff9e35da6f4b40 x1874672085873920/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829656 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829631.496703:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4b40 x1874672085873920/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829656 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829631.496727:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829631.496731:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829631.496737:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085873920:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829631.496745:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085874048:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829631.496766:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829631.496781:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829631/real 1787829631] req@ffff9e35da6f70c0 x1874672085874048/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829656 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829631.496799:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f70c0 x1874672085874048/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829656 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829631.496812:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829631.496816:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829631.496822:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085874048:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829631.496905:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085874176:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829631.496928:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829631.496940:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829631/real 1787829631] req@ffff9e35da6f7840 x1874672085874176/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829656 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829631.496953:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7840 x1874672085874176/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829656 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829631.496963:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829631.496967:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829631.496971:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085874176:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829631.496977:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085874304:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829631.496992:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829631.497002:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829631/real 1787829631] req@ffff9e35da6f61c0 x1874672085874304/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829656 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829631.497014:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f61c0 x1874672085874304/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829656 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829631.497023:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829631.497026:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829631.497029:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085874304:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829631.497035:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085874432:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829631.497049:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829631.497058:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829631/real 1787829631] req@ffff9e35da6f4f00 x1874672085874432/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829656 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829631.497070:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4f00 x1874672085874432/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829656 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829631.497079:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829631.497082:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829631.497085:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085874432:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829631.623931:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829631.623931:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829631.623932:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787829631.624110:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829632.647879:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829632.647884:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829632.647895:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787829632.647980:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829633.671918:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829633.671923:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829633.671924:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829633.671978:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829634.695957:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829634.695957:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829634.695957:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787829634.696014:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829635.719902:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829635.719926:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829635.719942:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787829635.719965:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829636.615930:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829636.615941:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829636.615945:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829636.615955:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3071 00000100:00080000:2.0:1787829636.615966:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829636.615973:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787829636.616016:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e1c0 x1874672085874560/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829636.616044:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829636.616048:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829636.616050:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829636.616055:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3071 00000100:00080000:2.0:1787829636.616059:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829636.616064:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829636.616075:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829636.616090:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085874560:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1787829636.616091:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c780 x1874672085874688/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829636.616107:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829636.616111:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829636.616113:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829636.616118:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3071 00000100:00080000:2.0:1787829636.616122:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829636.616126:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829636.616132:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829636.616144:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c3c0 x1874672085874816/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829636.616155:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1787829636.616156:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1787829636.616162:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829636.616165:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829636.616172:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3071 00000100:00080000:2.0:1787829636.616177:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829636.616181:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829636.616187:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:0.0:1787829636.616197:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829636/real 1787829636] req@ffff9e35c498e1c0 x1874672085874560/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829661 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829636.616203:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cf00 x1874672085874944/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829636.616217:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e1c0 x1874672085874560/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829661 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829636.616218:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829636.616228:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829636.616234:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829636.616236:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829636.616241:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829636.616243:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085874560:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1787829636.616250:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3071 00000100:00080000:2.0:1787829636.616256:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829636.616260:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829636.616267:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:0.0:1787829636.616268:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085874688:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829636.616285:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f840 x1874672085875072/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829636.616292:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829636.616304:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829636/real 1787829636] req@ffff9e35c498c780 x1874672085874688/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829661 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829636.616310:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787829636.616316:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c780 x1874672085874688/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829661 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829636.616317:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829636.616321:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829636.616331:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829636.616332:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3071 00000100:00080000:0.0:1787829636.616337:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:2.0:1787829636.616340:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787829636.616341:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085874688:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1787829636.616350:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787829636.616351:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085874816:192.168.203.141@tcp:38:kworker.0 00000008:00100000:2.0:1787829636.616361:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1787829636.616372:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829636.616380:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498ed00 x1874672085875200/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829636.616383:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829636/real 1787829636] req@ffff9e35c498c3c0 x1874672085874816/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829661 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829636.616404:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c3c0 x1874672085874816/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829661 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829636.616421:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829636.616421:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829636.616428:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:2.0:1787829636.616428:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829636.616432:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787829636.616435:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085874816:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1787829636.616440:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3071 00000100:00100000:0.0:1787829636.616444:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085874944:192.168.203.141@tcp:8:kworker.0 00000100:00080000:2.0:1787829636.616445:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829636.616453:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829636.616458:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1787829636.616466:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829636.616473:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498de00 x1874672085875328/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829636.616478:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829636/real 1787829636] req@ffff9e35c498cf00 x1874672085874944/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829661 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829636.616489:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829636.616494:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829636.616495:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829636.616498:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3071 00000100:00080000:2.0:1787829636.616500:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829636.616505:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829636.616509:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:0.0:1787829636.616513:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498cf00 x1874672085874944/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829661 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829636.616519:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e1c0 x1874672085875456/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829636.616525:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829636.616533:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829636.616536:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:2.0:1787829636.616538:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829636.616540:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829636.616546:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3071 00000100:00100000:0.0:1787829636.616547:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085874944:192.168.203.141@tcp:8:kworker.0 00000100:00080000:2.0:1787829636.616549:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829636.616555:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829636.616559:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:2.0:1787829636.616567:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f480 x1874672085875584/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829636.616584:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085875072:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829636.616606:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829636.616614:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829636/real 1787829636] req@ffff9e35c498f840 x1874672085875072/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829661 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829636.616627:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f840 x1874672085875072/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829661 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829636.616633:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829636.616635:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829636.616637:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085875072:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829636.616641:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085875200:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829636.616651:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829636.616657:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829636/real 1787829636] req@ffff9e35c498ed00 x1874672085875200/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829661 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829636.616665:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498ed00 x1874672085875200/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829661 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829636.616670:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829636.616671:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829636.616673:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085875200:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829636.616677:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085875328:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829636.616687:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829636.616692:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829636/real 1787829636] req@ffff9e35c498de00 x1874672085875328/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829661 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829636.616699:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498de00 x1874672085875328/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829661 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829636.616708:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829636.616710:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829636.616712:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085875328:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829636.616716:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085875456:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829636.616725:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829636.616730:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829636/real 1787829636] req@ffff9e35c498e1c0 x1874672085875456/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829661 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829636.616737:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e1c0 x1874672085875456/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829661 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829636.616742:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829636.616744:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829636.616746:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085875456:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829636.616749:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085875584:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829636.616758:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829636.616764:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829636/real 1787829636] req@ffff9e35c498f480 x1874672085875584/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829661 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829636.616771:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f480 x1874672085875584/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829661 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829636.616776:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829636.616778:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829636.616780:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085875584:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829636.743900:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829636.743900:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829636.743918:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787829636.743953:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829637.767893:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829637.767900:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829637.767904:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787829637.767927:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829638.791902:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829638.791902:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829638.791903:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787829638.791938:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829639.815924:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829639.815934:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787829639.815958:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829639.815967:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829640.839949:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829640.839949:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829640.840016:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787829640.840038:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829641.735941:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829641.735953:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829641.735958:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829641.735980:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3077 00000100:00080000:2.0:1787829641.735993:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829641.736001:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787829641.736040:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498d680 x1874672085875712/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829641.736073:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829641.736076:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829641.736078:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829641.736083:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3077 00000100:00080000:2.0:1787829641.736087:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829641.736091:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829641.736105:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829641.736119:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498fc00 x1874672085875840/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829641.736131:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829641.736135:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829641.736136:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829641.736143:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3077 00000100:00080000:2.0:1787829641.736147:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829641.736152:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829641.736157:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829641.736171:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085875712:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1787829641.736173:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cb40 x1874672085875968/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829641.736189:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829641.736192:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829641.736194:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829641.736202:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3077 00000100:00080000:2.0:1787829641.736205:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829641.736208:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829641.736212:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:2.0:1787829641.736222:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c000 x1874672085876096/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829641.736231:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829641.736235:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829641.736237:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829641.736242:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3077 00000100:00080000:2.0:1787829641.736248:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829641.736254:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829641.736258:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:2.0:1787829641.736267:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f0c0 x1874672085876224/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829641.736274:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829641.736283:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829641.736285:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829641.736292:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3077 00000100:00080000:2.0:1787829641.736299:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829641.736305:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829641.736310:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829641.736330:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e940 x1874672085876352/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829641.736342:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829641.736346:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829641.736351:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829641.736356:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3077 00000100:00080000:2.0:1787829641.736360:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829641.736363:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829641.736366:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829641.736376:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e580 x1874672085876480/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829641.736382:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829641.736385:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829641.736387:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829641.736390:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3077 00000100:00080000:2.0:1787829641.736392:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829641.736394:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829641.736397:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:2.0:1787829641.736404:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f480 x1874672085876608/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829641.736409:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829641.736412:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829641.736413:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829641.736416:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3077 00000100:00080000:2.0:1787829641.736418:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829641.736420:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829641.736422:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:2.0:1787829641.736429:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e1c0 x1874672085876736/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787829641.736485:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787829641.736575:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829641/real 1787829641] req@ffff9e35c498d680 x1874672085875712/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829666 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829641.736598:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498d680 x1874672085875712/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829666 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829641.736620:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829641.736627:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787829641.736636:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085875712:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1787829641.736647:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085875840:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787829641.736671:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787829641.736681:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829641/real 1787829641] req@ffff9e35c498fc00 x1874672085875840/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829666 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829641.736691:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498fc00 x1874672085875840/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829666 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829641.736697:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829641.736700:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787829641.736702:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085875840:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787829641.736726:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085875968:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787829641.736738:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787829641.736745:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829641/real 1787829641] req@ffff9e35c498cb40 x1874672085875968/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829666 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829641.736753:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498cb40 x1874672085875968/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829666 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829641.736776:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829641.736779:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787829641.736781:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085875968:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787829641.736786:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085876096:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787829641.736839:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787829641.736852:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829641/real 1787829641] req@ffff9e35c498c000 x1874672085876096/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829666 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829641.736868:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c000 x1874672085876096/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829666 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829641.736875:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829641.736878:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787829641.736881:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085876096:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787829641.736886:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085876224:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787829641.736898:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787829641.736904:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829641/real 1787829641] req@ffff9e35c498f0c0 x1874672085876224/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829666 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829641.736913:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f0c0 x1874672085876224/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829666 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829641.736922:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829641.736926:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787829641.736929:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085876224:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787829641.736935:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085876352:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787829641.736957:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787829641.736968:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829641/real 1787829641] req@ffff9e35c498e940 x1874672085876352/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829666 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829641.736982:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e940 x1874672085876352/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829666 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829641.736989:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829641.736991:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787829641.736993:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085876352:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787829641.736997:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085876480:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787829641.737013:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787829641.737021:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829641/real 1787829641] req@ffff9e35c498e580 x1874672085876480/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829666 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829641.737033:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e580 x1874672085876480/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829666 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829641.737043:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829641.737046:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787829641.737049:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085876480:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787829641.737056:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085876608:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787829641.737128:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787829641.737137:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829641/real 1787829641] req@ffff9e35c498f480 x1874672085876608/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829666 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829641.737145:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f480 x1874672085876608/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829666 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829641.737160:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829641.737164:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787829641.737167:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085876608:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787829641.737174:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085876736:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787829641.737195:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787829641.737207:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829641/real 1787829641] req@ffff9e35c498e1c0 x1874672085876736/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829666 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829641.737220:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e1c0 x1874672085876736/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829666 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829641.737229:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829641.737233:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787829641.737236:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085876736:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829641.863899:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829641.863949:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829641.863981:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787829641.863997:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829642.887940:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829642.887941:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829642.887953:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787829642.888007:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829643.911960:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829643.911994:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829643.911998:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787829643.912017:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829644.935880:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829644.935908:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829644.935910:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787829644.935960:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829645.959879:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829645.959938:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829645.959938:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829645.959954:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00080000:0.0:1787829646.855889:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829646.855897:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787829646.855902:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829646.855910:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3082 00000100:00080000:0.0:1787829646.855919:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829646.855925:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787829646.855956:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718780 x1874672085876864/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829646.855979:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829646.855982:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829646.855984:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829646.855986:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3082 00000100:00080000:0.0:1787829646.855989:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829646.855991:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829646.856000:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829646.856011:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718f00 x1874672085876992/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829646.856020:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1787829646.856023:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085876864:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1787829646.856024:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787829646.856028:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829646.856035:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3082 00000100:00080000:0.0:1787829646.856037:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829646.856039:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829646.856043:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829646.856051:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a940 x1874672085877120/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829646.856057:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829646.856060:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829646.856062:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829646.856064:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3082 00000100:00080000:0.0:1787829646.856066:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829646.856069:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829646.856072:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:0.0:1787829646.856079:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7183c0 x1874672085877248/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829646.856085:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829646.856088:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829646.856089:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829646.856095:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3082 00000800:00000100:3.0:1787829646.856099:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787829646.856101:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829646.856106:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829646.856109:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:0.0:1787829646.856118:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da719a40 x1874672085877376/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829646.856125:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829646.856128:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829646.856129:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829646.856133:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3082 00000100:00080000:0.0:1787829646.856136:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1787829646.856140:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829646/real 1787829646] req@ffff9e35da718780 x1874672085876864/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829671 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829646.856142:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829646.856149:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829646.856163:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a580 x1874672085877504/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787829646.856167:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718780 x1874672085876864/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829671 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829646.856180:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829646.856184:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:0.0:1787829646.856187:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1787829646.856189:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085876864:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1787829646.856198:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787829646.856201:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829646.856206:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3082 00000100:00080000:0.0:1787829646.856211:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1787829646.856212:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085876992:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787829646.856219:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829646.856227:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1787829646.856236:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829646.856244:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718000 x1874672085877632/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787829646.856250:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829646/real 1787829646] req@ffff9e35da718f00 x1874672085876992/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829671 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829646.856261:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829646.856265:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829646.856268:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1787829646.856268:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672085876992/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829671 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829646.856278:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3082 00000100:00080000:0.0:1787829646.856286:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829646.856287:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829646.856292:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:3.0:1787829646.856293:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787829646.856300:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085876992:192.168.203.141@tcp:38:kworker.0 00000008:00100000:0.0:1787829646.856301:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:3.0:1787829646.856308:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085877120:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829646.856314:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718b40 x1874672085877760/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829646.856322:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829646.856324:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829646.856327:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:3.0:1787829646.856328:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787829646.856332:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3082 00000100:00080000:0.0:1787829646.856336:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1787829646.856340:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829646/real 1787829646] req@ffff9e35da71a940 x1874672085877120/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829671 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829646.856341:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829646.856365:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:3.0:1787829646.856377:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672085877120/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829671 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829646.856381:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a1c0 x1874672085877888/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829646.856400:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829646.856408:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787829646.856413:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085877120:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787829646.856422:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085877248:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787829646.856462:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787829646.856474:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829646/real 1787829646] req@ffff9e35da7183c0 x1874672085877248/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829671 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829646.856483:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672085877248/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829671 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829646.856489:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829646.856491:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787829646.856493:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085877248:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787829646.856497:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085877376:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787829646.856511:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787829646.856517:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829646/real 1787829646] req@ffff9e35da719a40 x1874672085877376/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829671 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829646.856524:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672085877376/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829671 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829646.856530:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829646.856532:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787829646.856534:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085877376:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787829646.856537:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085877504:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787829646.856547:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787829646.856552:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829646/real 1787829646] req@ffff9e35da71a580 x1874672085877504/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829671 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829646.856559:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672085877504/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829671 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829646.856565:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829646.856567:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787829646.856569:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085877504:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787829646.856584:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085877632:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787829646.856595:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787829646.856601:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829646/real 1787829646] req@ffff9e35da718000 x1874672085877632/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829671 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829646.856608:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718000 x1874672085877632/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829671 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829646.856613:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829646.856615:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787829646.856617:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085877632:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787829646.856621:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085877760:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787829646.856630:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787829646.856636:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829646/real 1787829646] req@ffff9e35da718b40 x1874672085877760/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829671 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829646.856642:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718b40 x1874672085877760/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829671 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829646.856648:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829646.856650:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787829646.856652:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085877760:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787829646.856656:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085877888:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787829646.856665:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787829646.856670:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829646/real 1787829646] req@ffff9e35da71a1c0 x1874672085877888/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829671 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829646.856677:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a1c0 x1874672085877888/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829671 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829646.856682:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829646.856684:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787829646.856686:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085877888:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829646.983905:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829646.983905:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829646.983905:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787829646.984090:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829648.007863:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829648.007872:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829648.007886:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787829648.007927:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829649.031896:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829649.031896:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829649.031910:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829649.031957:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829650.055865:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829650.055891:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829650.055908:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787829650.055963:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829651.079898:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829651.079898:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829651.079931:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787829651.079977:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829651.975953:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829651.975975:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829651.975979:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829651.975990:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3087 00000100:00080000:2.0:1787829651.976001:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829651.976009:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787829651.976049:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498de00 x1874672085878016/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829651.976072:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829651.976076:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829651.976079:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829651.976084:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3087 00000100:00080000:2.0:1787829651.976088:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829651.976096:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787829651.976099:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085878016:192.168.203.141@tcp:250:kworker.0 00000008:00100000:2.0:1787829651.976108:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829651.976129:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498ed00 x1874672085878144/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829651.976152:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829651.976156:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829651.976159:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829651.976164:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3087 00000100:00080000:2.0:1787829651.976172:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829651.976177:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829651.976184:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829651.976197:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f840 x1874672085878272/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829651.976207:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829651.976210:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829651.976213:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829651.976218:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3087 00000100:00080000:2.0:1787829651.976224:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829651.976231:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787829651.976238:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085878144:192.168.203.141@tcp:38:kworker.0 00000008:00100000:2.0:1787829651.976239:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:2.0:1787829651.976265:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cf00 x1874672085878400/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829651.976282:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1787829651.976282:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085878272:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1787829651.976296:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829651.976300:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829651.976309:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3087 00000100:00080000:2.0:1787829651.976317:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1787829651.976320:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085878400:192.168.203.141@tcp:8:kworker.0 00000100:00080000:2.0:1787829651.976328:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829651.976339:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:2.0:1787829651.976359:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c3c0 x1874672085878528/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829651.976378:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829651.976383:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829651.976386:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829651.976392:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3087 00000100:00080000:2.0:1787829651.976397:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1787829651.976399:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085878528:192.168.203.141@tcp:8:kworker.0 00000100:00080000:2.0:1787829651.976409:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829651.976417:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829651.976437:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c780 x1874672085878656/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829651.976457:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829651.976462:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829651.976465:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1787829651.976475:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085878656:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1787829651.976477:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3087 00000100:00080000:2.0:1787829651.976490:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829651.976497:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829651.976503:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829651.976517:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e1c0 x1874672085878784/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829651.976533:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829651.976538:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829651.976541:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1787829651.976550:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085878784:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1787829651.976552:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3087 00000100:00080000:2.0:1787829651.976567:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829651.976575:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829651.976582:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:2.0:1787829651.976598:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f480 x1874672085878912/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829651.976621:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829651.976626:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829651.976627:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829651.976632:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3087 00000100:00080000:2.0:1787829651.976638:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829651.976643:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829651.976649:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:3.0:1787829651.976650:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085878912:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829651.976676:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e580 x1874672085879040/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787829651.976731:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085879040:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829652.103861:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829652.103929:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829652.103938:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829652.103954:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829653.127922:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829653.127924:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829653.127938:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787829653.127971:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829654.151952:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829654.151955:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829654.151977:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787829654.152017:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1787829655.048123:0:2398:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.141:988 00000400:00000100:3.0:1787829655.048172:0:2398:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.141@tcp at host 192.168.203.141 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1787829655.048192:0:2398:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1787829655.048254:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000800:00000100:3.0:1787829655.048269:0:2398:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1787829655.048279:0:2398:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1787829655.048290:0:2398:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:0.0:1787829655.048291:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829651/real 1787829655] req@ffff9e35c498e580 x1874672085879040/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829676 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:3.0:1787829655.048303:0:2398:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1787829655.048320:0:2398:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:0.0:1787829655.048331:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e580 x1874672085879040/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829676 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:3.0:1787829655.048337:0:2398:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1787829655.048354:0:2398:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00080000:0.0:1787829655.048355:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829655.048365:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829655.048370:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085879040:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829655.048379:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829651/real 1787829655] req@ffff9e35c498f480 x1874672085878912/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829676 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829655.048400:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f480 x1874672085878912/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829676 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829655.048408:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829655.048412:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829655.048415:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085878912:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829655.048421:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829651/real 1787829655] req@ffff9e35c498e1c0 x1874672085878784/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829676 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829655.048432:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e1c0 x1874672085878784/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829676 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829655.048439:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829655.048442:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829655.048446:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085878784:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829655.048452:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829651/real 1787829655] req@ffff9e35c498c780 x1874672085878656/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829676 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829655.048465:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c780 x1874672085878656/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829676 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829655.048474:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829655.048477:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829655.048480:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085878656:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829655.048487:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829651/real 1787829655] req@ffff9e35c498c3c0 x1874672085878528/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829676 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829655.048497:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c3c0 x1874672085878528/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829676 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829655.048505:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829655.048509:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829655.048513:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085878528:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829655.048520:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829651/real 1787829655] req@ffff9e35c498cf00 x1874672085878400/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829676 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829655.048533:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498cf00 x1874672085878400/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829676 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829655.048542:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829655.048546:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829655.048549:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085878400:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829655.048555:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829651/real 1787829655] req@ffff9e35c498ed00 x1874672085878144/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829676 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829655.048575:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498ed00 x1874672085878144/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829676 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829655.048585:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829655.048589:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829655.048593:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085878144:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829655.048600:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829651/real 1787829655] req@ffff9e35c498f840 x1874672085878272/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829676 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829655.048612:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f840 x1874672085878272/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829676 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829655.048619:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829655.048623:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829655.048626:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085878272:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829655.048632:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829651/real 1787829655] req@ffff9e35c498de00 x1874672085878016/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829676 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829655.048643:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498de00 x1874672085878016/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829676 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829655.048651:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829655.048655:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829655.048657:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085878016:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1787829655.175910:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829655.175910:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829655.175916:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829655.176046:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829656.199878:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829656.199878:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829656.199879:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787829656.199910:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829657.095938:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829657.095948:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829657.095952:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829657.095961:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3092 00000100:00080000:1.0:1787829657.095972:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829657.095980:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787829657.096016:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6580 x1874672085879168/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829657.096041:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829657.096044:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829657.096045:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829657.096056:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3092 00000100:00080000:1.0:1787829657.096059:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829657.096062:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829657.096072:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829657.096084:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7480 x1874672085879296/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829657.096095:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829657.096103:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:0.0:1787829657.096106:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085879168:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787829657.096106:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829657.096121:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3092 00000100:00080000:1.0:1787829657.096127:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829657.096132:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829657.096138:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829657.096156:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4000 x1874672085879424/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829657.096166:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829657.096169:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829657.096171:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829657.096177:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3092 00000100:00080000:1.0:1787829657.096194:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829657.096197:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829657.096202:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:1.0:1787829657.096211:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6940 x1874672085879552/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829657.096224:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787829657.096232:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829657.096236:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829657.096237:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829657.096246:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3092 00000100:00080000:1.0:1787829657.096253:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829657.096256:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829657.096261:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:1.0:1787829657.096271:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6d00 x1874672085879680/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829657.096279:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829657.096282:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829657.096283:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829657.096288:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3092 00000100:00080000:1.0:1787829657.096294:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829657.096300:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829657.096308:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829657.096309:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829657/real 1787829657] req@ffff9e35da6f6580 x1874672085879168/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829682 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829657.096336:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4780 x1874672085879808/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829657.096344:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6580 x1874672085879168/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829682 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829657.096375:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829657.096380:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829657.096386:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:1.0:1787829657.096392:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:0.0:1787829657.096399:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085879168:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787829657.096400:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829657.096419:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3092 00000100:00080000:1.0:1787829657.096426:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829657.096432:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829657.096439:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829657.096456:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085879296:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829657.096467:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6580 x1874672085879936/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829657.096491:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829657.096496:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000100:0.0:1787829657.096504:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787829657.096506:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829657.096520:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3092 00000100:00080000:1.0:1787829657.096526:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787829657.096534:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829657/real 1787829657] req@ffff9e35da6f7480 x1874672085879296/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829682 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829657.096536:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829657.096553:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:0.0:1787829657.096564:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7480 x1874672085879296/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829682 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829657.096578:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4f00 x1874672085880064/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829657.096579:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829657.096591:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829657.096605:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085879296:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787829657.096630:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787829657.096637:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085879424:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787829657.096639:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829657.096649:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829657.096673:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3092 00000100:00080000:1.0:1787829657.096680:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829657.096686:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829657.096693:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000800:00000100:0.0:1787829657.096697:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829657.096730:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829657/real 1787829657] req@ffff9e35da6f4000 x1874672085879424/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829682 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829657.096731:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f61c0 x1874672085880192/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829657.096870:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4000 x1874672085879424/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829682 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829657.096884:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829657.096902:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829657.096907:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085879424:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829657.096921:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085879552:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829657.096949:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829657.096973:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829657/real 1787829657] req@ffff9e35da6f6940 x1874672085879552/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829682 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829657.096988:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6940 x1874672085879552/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829682 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829657.096997:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829657.097001:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829657.097004:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085879552:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829657.097012:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085879680:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829657.097034:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829657.097047:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829657/real 1787829657] req@ffff9e35da6f6d00 x1874672085879680/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829682 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829657.097060:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6d00 x1874672085879680/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829682 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829657.097070:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829657.097074:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829657.097077:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085879680:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829657.097084:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085879808:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829657.097114:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829657.097126:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829657/real 1787829657] req@ffff9e35da6f4780 x1874672085879808/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829682 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829657.097145:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4780 x1874672085879808/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829682 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829657.097155:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829657.097158:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829657.097162:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085879808:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829657.097189:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085879936:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829657.097213:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829657.097225:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829657/real 1787829657] req@ffff9e35da6f6580 x1874672085879936/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829682 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829657.097238:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6580 x1874672085879936/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829682 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829657.097252:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829657.097255:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829657.097259:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085879936:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829657.097266:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085880064:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829657.097284:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829657.097296:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829657/real 1787829657] req@ffff9e35da6f4f00 x1874672085880064/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829682 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829657.097306:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4f00 x1874672085880064/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829682 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829657.097315:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829657.097318:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829657.097321:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085880064:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829657.097328:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085880192:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829657.097346:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829657.097357:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829657/real 1787829657] req@ffff9e35da6f61c0 x1874672085880192/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829682 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829657.097370:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f61c0 x1874672085880192/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829682 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829657.097379:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829657.097382:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829657.097387:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085880192:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829657.223973:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829657.223973:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829657.223974:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829657.224086:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829658.247924:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829658.247924:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829658.247958:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829658.248111:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829659.271895:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829659.271895:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829659.271928:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829659.271942:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829660.295917:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829660.295920:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829660.295923:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829660.295985:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829661.319915:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829661.319949:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829661.319976:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829661.319983:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00080000:3.0:1787829662.215935:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829662.215944:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787829662.215949:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829662.215959:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3097 00000100:00080000:3.0:1787829662.215970:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829662.215977:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787829662.216022:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a1c0 x1874672085880320/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829662.216050:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829662.216053:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829662.216054:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829662.216058:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3097 00000100:00080000:3.0:1787829662.216060:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829662.216063:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829662.216073:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829662.216083:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498b40 x1874672085880448/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829662.216091:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829662.216094:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787829662.216096:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829662.216099:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3097 00000100:00080000:3.0:1787829662.216104:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787829662.216105:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085880320:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1787829662.216108:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829662.216115:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829662.216122:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649bc00 x1874672085880576/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829662.216128:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829662.216131:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787829662.216132:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829662.216135:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3097 00000100:00080000:3.0:1787829662.216138:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829662.216141:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829662.216145:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:3.0:1787829662.216153:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b840 x1874672085880704/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829662.216159:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829662.216162:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787829662.216163:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829662.216166:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3097 00000100:00080000:3.0:1787829662.216168:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829662.216170:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829662.216173:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:3.0:1787829662.216181:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499680 x1874672085880832/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829662.216188:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1787829662.216190:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829662.216194:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829662.216196:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829662.216200:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3097 00000100:00080000:3.0:1787829662.216203:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829662.216205:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829662.216209:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829662.216216:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a940 x1874672085880960/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829662.216227:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829662.216229:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787829662.216231:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829662.216234:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3097 00000100:00080000:3.0:1787829662.216236:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787829662.216238:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829662/real 1787829662] req@ffff9e35c649a1c0 x1874672085880320/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829687 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829662.216241:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829662.216248:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829662.216260:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b0c0 x1874672085881088/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829662.216266:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a1c0 x1874672085880320/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829687 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829662.216270:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829662.216277:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787829662.216280:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829662.216286:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3097 00000100:00080000:0.0:1787829662.216289:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829662.216289:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829662.216294:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829662.216297:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085880320:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1787829662.216303:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829662.216307:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:3.0:1787829662.216315:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499e00 x1874672085881216/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829662.216321:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829662.216324:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787829662.216325:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787829662.216329:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085880448:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1787829662.216332:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3097 00000100:00080000:3.0:1787829662.216338:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829662.216341:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829662.216345:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:3.0:1787829662.216353:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a580 x1874672085881344/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829662.216357:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829662.216372:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829662/real 1787829662] req@ffff9e35c6498b40 x1874672085880448/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829687 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829662.216382:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498b40 x1874672085880448/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829687 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829662.216390:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829662.216393:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829662.216395:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085880448:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829662.216400:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085880576:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829662.216412:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829662.216419:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829662/real 1787829662] req@ffff9e35c649bc00 x1874672085880576/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829687 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829662.216427:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649bc00 x1874672085880576/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829687 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829662.216432:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829662.216434:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829662.216436:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085880576:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829662.216440:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085880704:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829662.216451:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829662.216456:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829662/real 1787829662] req@ffff9e35c649b840 x1874672085880704/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829687 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829662.216464:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b840 x1874672085880704/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829687 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829662.216470:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829662.216471:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829662.216473:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085880704:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829662.216477:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085880832:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829662.216487:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829662.216493:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829662/real 1787829662] req@ffff9e35c6499680 x1874672085880832/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829687 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829662.216500:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499680 x1874672085880832/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829687 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829662.216506:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829662.216508:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829662.216510:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085880832:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829662.216513:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085880960:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829662.216523:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829662.216529:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829662/real 1787829662] req@ffff9e35c649a940 x1874672085880960/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829687 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829662.216537:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a940 x1874672085880960/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829687 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829662.216542:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829662.216544:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829662.216546:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085880960:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829662.216549:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085881088:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829662.216559:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829662.216564:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829662/real 1787829662] req@ffff9e35c649b0c0 x1874672085881088/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829687 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829662.216572:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b0c0 x1874672085881088/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829687 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829662.216582:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829662.216584:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829662.216586:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085881088:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829662.216589:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085881216:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829662.216599:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829662.216604:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829662/real 1787829662] req@ffff9e35c6499e00 x1874672085881216/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829687 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829662.216611:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499e00 x1874672085881216/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829687 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829662.216632:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829662.216635:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829662.216638:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085881216:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829662.216655:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085881344:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829662.216671:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829662.216678:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829662/real 1787829662] req@ffff9e35c649a580 x1874672085881344/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829687 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829662.216686:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a580 x1874672085881344/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829687 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829662.216692:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829662.216694:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829662.216697:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085881344:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829662.343953:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829662.343963:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829662.343994:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829662.344039:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829663.367895:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:3.0:1787829663.367895:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829663.367902:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829663.367976:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829664.391924:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829664.391934:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829664.391971:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829664.391999:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829665.415931:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829665.415932:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787829665.415987:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829665.416063:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829666.439941:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829666.439980:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829666.440070:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829666.440111:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00080000:2.0:1787829667.335936:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829667.335944:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829667.335947:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829667.335956:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3102 00000100:00080000:2.0:1787829667.335965:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829667.335971:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787829667.336001:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e940 x1874672085881472/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829667.336027:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829667.336030:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829667.336032:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829667.336035:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3102 00000100:00080000:2.0:1787829667.336037:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829667.336039:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829667.336047:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829667.336067:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f0c0 x1874672085881600/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829667.336075:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829667.336077:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829667.336078:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829667.336081:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3102 00000100:00080000:2.0:1787829667.336083:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787829667.336088:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085881472:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1787829667.336088:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829667.336095:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829667.336102:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c000 x1874672085881728/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829667.336109:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829667.336111:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829667.336113:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829667.336115:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3102 00000100:00080000:2.0:1787829667.336118:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829667.336120:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829667.336123:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:2.0:1787829667.336131:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cb40 x1874672085881856/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829667.336140:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829667.336143:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829667.336145:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1787829667.336151:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1787829667.336152:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3102 00000100:00080000:2.0:1787829667.336160:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829667.336165:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829667.336171:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:2.0:1787829667.336186:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498fc00 x1874672085881984/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829667.336192:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829667/real 1787829667] req@ffff9e35c498e940 x1874672085881472/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829692 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829667.336199:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829667.336206:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829667.336210:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829667.336215:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3102 00000100:00080000:2.0:1787829667.336218:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787829667.336219:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e940 x1874672085881472/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829692 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829667.336224:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829667.336233:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1787829667.336236:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829667.336241:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829667.336245:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498d680 x1874672085882112/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829667.336248:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085881472:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1787829667.336262:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085881600:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1787829667.336273:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829667.336275:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829667.336276:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829667.336279:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3102 00000100:00080000:2.0:1787829667.336282:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829667.336284:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829667.336288:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1787829667.336296:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829667.336298:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498de00 x1874672085882240/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829667.336306:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787829667.336311:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829667/real 1787829667] req@ffff9e35c498f0c0 x1874672085881600/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829692 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829667.336312:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829667.336315:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829667.336322:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3102 00000100:00080000:2.0:1787829667.336327:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829667.336332:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829667.336336:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:0.0:1787829667.336342:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f0c0 x1874672085881600/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829692 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829667.336345:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f840 x1874672085882368/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829667.336367:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829667.336369:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829667.336370:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829667.336374:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3102 00000100:00080000:2.0:1787829667.336379:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829667.336383:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:0.0:1787829667.336385:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000008:00100000:2.0:1787829667.336386:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00080000:0.0:1787829667.336396:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829667.336397:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498ed00 x1874672085882496/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829667.336406:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085881600:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829667.336439:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085881728:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829667.336457:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829667.336466:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829667/real 1787829667] req@ffff9e35c498c000 x1874672085881728/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829692 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829667.336474:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c000 x1874672085881728/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829692 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829667.336480:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829667.336483:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829667.336485:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085881728:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829667.336489:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085881856:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829667.336499:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829667.336505:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829667/real 1787829667] req@ffff9e35c498cb40 x1874672085881856/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829692 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829667.336513:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498cb40 x1874672085881856/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829692 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829667.336518:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829667.336520:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829667.336522:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085881856:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829667.336525:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085881984:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829667.336535:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829667.336541:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829667/real 1787829667] req@ffff9e35c498fc00 x1874672085881984/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829692 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829667.336549:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498fc00 x1874672085881984/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829692 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829667.336554:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829667.336556:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829667.336558:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085881984:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829667.336561:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085882112:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829667.336571:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829667.336578:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829667/real 1787829667] req@ffff9e35c498d680 x1874672085882112/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829692 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829667.336599:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498d680 x1874672085882112/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829692 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829667.336606:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829667.336608:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829667.336610:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085882112:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829667.336614:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085882240:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829667.336628:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829667.336634:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829667/real 1787829667] req@ffff9e35c498de00 x1874672085882240/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829692 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829667.336641:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498de00 x1874672085882240/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829692 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829667.336651:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829667.336653:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829667.336656:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085882240:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829667.336659:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085882368:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829667.336669:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829667.336674:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829667/real 1787829667] req@ffff9e35c498f840 x1874672085882368/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829692 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829667.336681:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f840 x1874672085882368/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829692 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829667.336686:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829667.336688:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829667.336690:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085882368:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829667.336693:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085882496:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829667.336702:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829667.336709:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829667/real 1787829667] req@ffff9e35c498ed00 x1874672085882496/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829692 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829667.336721:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498ed00 x1874672085882496/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829692 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829667.336729:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829667.336733:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829667.336736:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085882496:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829667.463915:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829667.463918:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829667.463928:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829667.463947:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829668.487939:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829668.487939:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829668.487939:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787829668.488015:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829669.511881:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829669.511906:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829669.512061:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829669.512064:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829670.535929:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829670.535959:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829670.536050:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829670.536085:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829671.559882:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829671.559933:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829671.559933:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829671.559950:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829672.455979:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829672.455989:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829672.455995:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829672.456009:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3107 00000100:00080000:1.0:1787829672.456022:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829672.456032:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787829672.456074:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7840 x1874672085882624/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829672.456134:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829672.456140:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829672.456143:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829672.456150:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3107 00000100:00080000:1.0:1787829672.456156:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829672.456163:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829672.456177:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829672.456188:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085882624:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1787829672.456204:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f70c0 x1874672085882752/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829672.456239:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829672.456242:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829672.456244:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829672.456248:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3107 00000100:00080000:1.0:1787829672.456266:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829672.456270:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829672.456296:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1787829672.456297:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829672.456322:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4b40 x1874672085882880/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829672.456336:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787829672.456356:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829672/real 1787829672] req@ffff9e35da6f7840 x1874672085882624/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829697 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829672.456360:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829672.456366:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829672.456376:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3107 00000100:00080000:1.0:1787829672.456382:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787829672.456386:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7840 x1874672085882624/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829697 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829672.456392:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829672.456404:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00080000:0.0:1787829672.456407:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829672.456414:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829672.456418:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085882624:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1787829672.456424:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5e00 x1874672085883008/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829672.456435:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829672.456438:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829672.456440:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787829672.456449:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085882752:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787829672.456449:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3107 00000100:00080000:1.0:1787829672.456473:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829672.456489:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829672.456495:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:1.0:1787829672.456510:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5a40 x1874672085883136/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829672.456516:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787829672.456530:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787829672.456535:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829672/real 1787829672] req@ffff9e35da6f70c0 x1874672085882752/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829697 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829672.456536:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829672.456541:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829672.456584:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3107 00000100:00100000:0.0:1787829672.456589:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f70c0 x1874672085882752/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829697 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829672.456590:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829672.456605:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:0.0:1787829672.456630:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000008:00100000:1.0:1787829672.456631:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1787829672.456638:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829672.456642:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085882752:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829672.456647:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085882880:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829672.456654:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f43c0 x1874672085883264/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829672.456667:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1787829672.456668:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787829672.456674:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829672.456676:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829672.456682:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3107 00000100:00100000:0.0:1787829672.456683:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829672/real 1787829672] req@ffff9e35da6f4b40 x1874672085882880/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829697 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829672.456688:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829672.456696:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787829672.456701:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4b40 x1874672085882880/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829697 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1787829672.456702:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1787829672.456714:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829672.456719:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829672.456720:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7840 x1874672085883392/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829672.456723:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085882880:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829672.456735:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085883008:192.168.203.141@tcp:8:kworker.0 00000100:00080000:1.0:1787829672.456739:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829672.456743:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829672.456745:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829672.456749:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3107 00000100:00080000:1.0:1787829672.456753:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1787829672.456757:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787829672.456762:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829672.456767:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:0.0:1787829672.456771:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829672/real 1787829672] req@ffff9e35da6f5e00 x1874672085883008/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829697 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829672.456780:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f61c0 x1874672085883520/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829672.456786:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5e00 x1874672085883008/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829697 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829672.456795:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829672.456798:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829672.456799:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829672.456802:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3107 00000100:00080000:1.0:1787829672.456805:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829672.456808:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829672.456811:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00080000:0.0:1787829672.456838:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829672.456845:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829672.456852:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085883008:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829672.456855:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4f00 x1874672085883648/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829672.456887:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085883136:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829672.456907:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829672.456916:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829672/real 1787829672] req@ffff9e35da6f5a40 x1874672085883136/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829697 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829672.456924:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5a40 x1874672085883136/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829697 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829672.456930:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829672.456932:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829672.456934:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085883136:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829672.456938:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085883264:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829672.456949:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829672.456955:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829672/real 1787829672] req@ffff9e35da6f43c0 x1874672085883264/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829697 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829672.456962:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f43c0 x1874672085883264/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829697 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829672.456968:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829672.456969:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829672.456971:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085883264:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829672.456975:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085883392:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829672.456985:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829672.456991:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829672/real 1787829672] req@ffff9e35da6f7840 x1874672085883392/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829697 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829672.457003:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7840 x1874672085883392/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829697 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829672.457008:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829672.457010:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829672.457011:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085883392:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829672.457015:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085883520:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829672.457025:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829672.457030:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829672/real 1787829672] req@ffff9e35da6f61c0 x1874672085883520/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829697 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829672.457038:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f61c0 x1874672085883520/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829697 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829672.457043:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829672.457045:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829672.457047:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085883520:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829672.457051:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085883648:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829672.457060:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829672.457066:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829672/real 1787829672] req@ffff9e35da6f4f00 x1874672085883648/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829697 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829672.457073:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4f00 x1874672085883648/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829697 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829672.457082:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829672.457084:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829672.457086:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085883648:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829672.583915:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829672.583929:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829672.583964:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829672.584049:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829673.607930:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829673.607931:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829673.607944:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829673.607977:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829674.631973:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829674.631999:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829674.632009:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829674.632030:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829675.655934:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829675.655940:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829675.655986:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829675.656003:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829676.679906:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829676.679939:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829676.679964:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829676.679977:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829677.575917:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829677.575926:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829677.575929:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829677.575939:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3112 00000100:00080000:2.0:1787829677.575948:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829677.575954:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787829677.575986:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cf00 x1874672085883776/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829677.576009:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829677.576012:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829677.576014:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829677.576017:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3112 00000100:00080000:2.0:1787829677.576020:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829677.576023:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829677.576032:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829677.576044:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c3c0 x1874672085883904/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829677.576053:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787829677.576056:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085883776:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1787829677.576057:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829677.576062:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829677.576067:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3112 00000100:00080000:2.0:1787829677.576070:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829677.576073:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829677.576077:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829677.576086:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c780 x1874672085884032/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829677.576094:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829677.576096:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829677.576098:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829677.576101:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3112 00000100:00080000:2.0:1787829677.576105:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829677.576108:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829677.576112:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000800:00000100:0.0:1787829677.576120:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829677.576122:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e1c0 x1874672085884160/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829677.576144:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829677.576150:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829677.576151:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829677.576157:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3112 00000100:00100000:0.0:1787829677.576160:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829677/real 1787829677] req@ffff9e35c498cf00 x1874672085883776/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829702 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829677.576162:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829677.576170:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829677.576178:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:0.0:1787829677.576186:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498cf00 x1874672085883776/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829702 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829677.576191:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f480 x1874672085884288/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829677.576203:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829677.576205:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829677.576210:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:2.0:1787829677.576211:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829677.576215:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787829677.576219:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085883776:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1787829677.576223:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3112 00000100:00080000:2.0:1787829677.576227:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829677.576230:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829677.576233:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829677.576240:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e580 x1874672085884416/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829677.576247:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829677.576250:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829677.576251:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787829677.576257:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085883904:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1787829677.576257:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3112 00000100:00080000:2.0:1787829677.576265:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829677.576269:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829677.576273:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829677.576283:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cf00 x1874672085884544/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829677.576288:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1787829677.576296:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829677.576299:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829677.576300:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829677.576305:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3112 00000100:00080000:2.0:1787829677.576307:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787829677.576311:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829677/real 1787829677] req@ffff9e35c498c3c0 x1874672085883904/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829702 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829677.576313:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829677.576318:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:2.0:1787829677.576335:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498ed00 x1874672085884672/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829677.576338:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c3c0 x1874672085883904/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829702 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829677.576364:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829677.576365:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829677.576372:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829677.576374:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:2.0:1787829677.576378:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787829677.576379:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085883904:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1787829677.576390:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3112 00000100:00100000:0.0:1787829677.576392:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085884032:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1787829677.576398:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829677.576406:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829677.576412:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000800:00000100:0.0:1787829677.576419:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829677.576428:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f840 x1874672085884800/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829677.576435:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829677/real 1787829677] req@ffff9e35c498c780 x1874672085884032/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829702 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829677.576482:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c780 x1874672085884032/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829702 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829677.576497:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829677.576502:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829677.576625:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085884032:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829677.576635:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085884160:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829677.576670:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829677.576684:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829677/real 1787829677] req@ffff9e35c498e1c0 x1874672085884160/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829702 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829677.576695:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e1c0 x1874672085884160/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829702 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829677.576701:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829677.576703:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829677.576705:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085884160:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829677.576716:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085884288:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829677.576732:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829677.576740:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829677/real 1787829677] req@ffff9e35c498f480 x1874672085884288/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829702 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829677.576747:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f480 x1874672085884288/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829702 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829677.576753:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829677.576755:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829677.576757:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085884288:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829677.576775:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085884416:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829677.576786:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829677.576793:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829677/real 1787829677] req@ffff9e35c498e580 x1874672085884416/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829702 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829677.576833:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e580 x1874672085884416/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829702 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829677.576844:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829677.576859:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829677.576863:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085884416:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829677.576869:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085884544:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829677.576893:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829677.576906:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829677/real 1787829677] req@ffff9e35c498cf00 x1874672085884544/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829702 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829677.576926:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498cf00 x1874672085884544/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829702 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829677.576935:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829677.576938:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829677.576941:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085884544:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829677.576948:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085884672:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829677.576969:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829677.576982:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829677/real 1787829677] req@ffff9e35c498ed00 x1874672085884672/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829702 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829677.576994:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498ed00 x1874672085884672/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829702 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829677.577003:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829677.577007:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829677.577010:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085884672:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829677.577016:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085884800:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829677.577036:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829677.577049:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829677/real 1787829677] req@ffff9e35c498f840 x1874672085884800/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829702 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829677.577063:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f840 x1874672085884800/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829702 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829677.577078:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829677.577082:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829677.577086:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085884800:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829677.703904:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829677.703928:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829677.703928:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829677.703933:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829678.727919:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829678.727919:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829678.727945:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829678.728005:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829679.751903:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829679.751904:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829679.751906:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829679.751972:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829680.775883:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829680.775895:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829680.775926:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787829680.775952:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829681.799900:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829681.799909:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829681.799937:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787829681.799951:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829682.695899:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829682.695907:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829682.695911:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829682.695920:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3118 00000100:00080000:1.0:1787829682.695929:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829682.695934:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787829682.695968:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6580 x1874672085884928/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829682.695994:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829682.695997:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829682.695998:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829682.696001:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3118 00000100:00080000:1.0:1787829682.696004:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829682.696006:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829682.696014:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829682.696024:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4780 x1874672085885056/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829682.696033:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829682.696036:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829682.696038:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829682.696043:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3118 00000100:00100000:0.0:1787829682.696045:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085884928:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787829682.696046:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829682.696053:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829682.696058:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829682.696066:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6d00 x1874672085885184/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829682.696072:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829682.696075:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829682.696076:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829682.696079:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3118 00000100:00080000:1.0:1787829682.696081:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829682.696083:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829682.696086:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:1.0:1787829682.696095:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6940 x1874672085885312/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829682.696102:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829682.696104:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829682.696105:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829682.696108:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3118 00000100:00080000:1.0:1787829682.696111:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1787829682.696113:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787829682.696116:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829682.696125:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:1.0:1787829682.696136:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4000 x1874672085885440/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829682.696143:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829682.696146:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829682.696147:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829682.696150:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3118 00000100:00080000:1.0:1787829682.696152:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829682.696155:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829682.696158:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829682.696166:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829682/real 1787829682] req@ffff9e35da6f6580 x1874672085884928/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829707 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829682.696168:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7480 x1874672085885568/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829682.696203:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829682.696205:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829682.696206:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829682.696211:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3118 00000100:00100000:0.0:1787829682.696212:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6580 x1874672085884928/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829707 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829682.696213:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829682.696220:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829682.696226:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1787829682.696232:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1787829682.696236:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4f00 x1874672085885696/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829682.696237:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829682.696254:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085884928:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787829682.696263:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829682.696266:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829682.696268:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829682.696272:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3118 00000100:00080000:1.0:1787829682.696275:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829682.696277:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829682.696280:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:1.0:1787829682.696289:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f61c0 x1874672085885824/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829682.696290:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085885056:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787829682.696304:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829682.696317:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829682.696319:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829682.696322:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3118 00000100:00080000:1.0:1787829682.696327:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1787829682.696330:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787829682.696335:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829682.696342:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:0.0:1787829682.696345:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829682/real 1787829682] req@ffff9e35da6f4780 x1874672085885056/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829707 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829682.696363:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4780 x1874672085885056/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829707 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829682.696364:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7840 x1874672085885952/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829682.696399:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829682.696403:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829682.696405:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085885056:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829682.696410:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085885184:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829682.696424:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829682.696430:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829682/real 1787829682] req@ffff9e35da6f6d00 x1874672085885184/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829707 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829682.696438:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6d00 x1874672085885184/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829707 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829682.696443:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829682.696445:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829682.696447:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085885184:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829682.696451:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085885312:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829682.696461:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829682.696466:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829682/real 1787829682] req@ffff9e35da6f6940 x1874672085885312/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829707 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829682.696486:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6940 x1874672085885312/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829707 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829682.696491:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829682.696494:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829682.696496:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085885312:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829682.696500:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085885440:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829682.696512:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829682.696518:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829682/real 1787829682] req@ffff9e35da6f4000 x1874672085885440/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829707 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829682.696525:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4000 x1874672085885440/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829707 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829682.696531:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829682.696532:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829682.696534:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085885440:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829682.696537:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085885568:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829682.696547:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829682.696553:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829682/real 1787829682] req@ffff9e35da6f7480 x1874672085885568/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829707 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829682.696560:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7480 x1874672085885568/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829707 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829682.696566:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829682.696567:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829682.696569:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085885568:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829682.696572:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085885696:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829682.696582:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829682.696589:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829682/real 1787829682] req@ffff9e35da6f4f00 x1874672085885696/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829707 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829682.696596:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4f00 x1874672085885696/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829707 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829682.696601:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829682.696602:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829682.696604:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085885696:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829682.696622:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085885824:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829682.696632:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829682.696639:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829682/real 1787829682] req@ffff9e35da6f61c0 x1874672085885824/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829707 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829682.696646:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f61c0 x1874672085885824/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829707 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829682.696651:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829682.696653:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829682.696655:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085885824:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829682.696658:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085885952:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829682.696667:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829682.696673:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829682/real 1787829682] req@ffff9e35da6f7840 x1874672085885952/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829707 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829682.696680:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7840 x1874672085885952/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829707 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829682.696686:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829682.696688:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829682.696690:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085885952:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829682.823885:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829682.823887:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829682.823921:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829682.823953:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829683.847906:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829683.847906:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829683.847933:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829683.847956:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829684.871907:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829684.871907:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829684.871910:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787829684.872001:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829685.895917:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829685.895919:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829685.895962:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829685.895985:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829686.919880:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829686.919883:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829686.919888:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829686.919929:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829687.815914:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829687.815926:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829687.815930:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829687.815940:0:27606:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3123 00000100:00080000:2.0:1787829687.815952:0:27606:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829687.815959:0:27606:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787829687.815999:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498de00 x1874672085886080/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829687.816029:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829687.816033:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829687.816035:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829687.816040:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3123 00000100:00080000:2.0:1787829687.816045:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829687.816050:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829687.816061:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829687.816071:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085886080:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1787829687.816079:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498d680 x1874672085886208/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829687.816093:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829687.816097:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829687.816100:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829687.816104:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3123 00000100:00080000:2.0:1787829687.816109:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829687.816114:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829687.816119:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1787829687.816131:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829687.816135:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498fc00 x1874672085886336/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829687.816147:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829687.816151:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829687.816153:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829687.816158:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3123 00000100:00080000:2.0:1787829687.816163:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829687.816167:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787829687.816170:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829687/real 1787829687] req@ffff9e35c498de00 x1874672085886080/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829712 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1787829687.816174:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:0.0:1787829687.816204:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498de00 x1874672085886080/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829712 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829687.816205:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cb40 x1874672085886464/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829687.816242:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829687.816242:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829687.816249:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:2.0:1787829687.816250:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829687.816255:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787829687.816256:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085886080:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1787829687.816264:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3123 00000100:00080000:2.0:1787829687.816269:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829687.816274:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829687.816279:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:0.0:1787829687.816284:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085886208:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829687.816293:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c000 x1874672085886592/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829687.816323:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1787829687.816331:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1787829687.816337:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829687.816344:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829687.816355:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3123 00000100:00100000:0.0:1787829687.816360:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829687/real 1787829687] req@ffff9e35c498d680 x1874672085886208/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829712 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829687.816367:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829687.816394:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787829687.816399:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498d680 x1874672085886208/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829712 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1787829687.816402:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1787829687.816414:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829687.816417:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829687.816423:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085886208:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829687.816424:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f0c0 x1874672085886720/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829687.816435:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085886336:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1787829687.816460:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829687.816466:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829687.816469:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829687.816489:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3123 00000100:00080000:2.0:1787829687.816495:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829687.816503:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:0.0:1787829687.816505:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:2.0:1787829687.816513:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829687.816525:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829687/real 1787829687] req@ffff9e35c498fc00 x1874672085886336/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829712 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829687.816532:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e940 x1874672085886848/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829687.816559:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498fc00 x1874672085886336/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829712 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829687.816568:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829687.816573:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829687.816575:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829687.816578:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:2.0:1787829687.816581:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787829687.816584:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085886336:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1787829687.816592:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3123 00000100:00080000:2.0:1787829687.816599:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787829687.816606:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085886464:192.168.203.141@tcp:8:kworker.0 00000100:00080000:2.0:1787829687.816608:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829687.816634:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000800:00000100:0.0:1787829687.816649:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829687.816651:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498de00 x1874672085886976/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829687.816661:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829687/real 1787829687] req@ffff9e35c498cb40 x1874672085886464/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829712 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829687.816666:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829687.816673:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:0.0:1787829687.816674:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498cb40 x1874672085886464/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829712 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829687.816678:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829687.816688:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829687.816689:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3123 00000100:00080000:0.0:1787829687.816694:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:2.0:1787829687.816697:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787829687.816699:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085886464:192.168.203.141@tcp:8:kworker.0 00000100:00080000:2.0:1787829687.816706:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829687.816713:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:2.0:1787829687.816726:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f840 x1874672085887104/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829687.816728:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085886592:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829687.816762:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829687.816773:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829687/real 1787829687] req@ffff9e35c498c000 x1874672085886592/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829712 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829687.816781:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c000 x1874672085886592/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829712 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829687.816789:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829687.816792:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829687.816795:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085886592:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829687.816801:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085886720:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829687.816852:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829687.816861:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829687/real 1787829687] req@ffff9e35c498f0c0 x1874672085886720/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829712 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829687.816870:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f0c0 x1874672085886720/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829712 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829687.816875:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829687.816877:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829687.816879:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085886720:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829687.816884:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085886848:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829687.816914:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829687.816926:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829687/real 1787829687] req@ffff9e35c498e940 x1874672085886848/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829712 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829687.816938:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e940 x1874672085886848/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829712 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829687.816947:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829687.816949:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829687.816951:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085886848:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829687.816955:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085886976:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829687.816970:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829687.816978:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829687/real 1787829687] req@ffff9e35c498de00 x1874672085886976/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829712 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829687.816985:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498de00 x1874672085886976/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829712 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829687.816991:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829687.816993:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829687.816994:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085886976:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829687.817008:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085887104:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829687.817018:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829687.817024:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829687/real 1787829687] req@ffff9e35c498f840 x1874672085887104/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829712 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829687.817031:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f840 x1874672085887104/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829712 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829687.817036:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829687.817039:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829687.817040:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085887104:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829687.943900:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829687.943901:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829687.943917:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787829687.943955:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829688.967889:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829688.967897:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829688.967913:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829688.967957:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829689.991900:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829689.991901:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829689.991926:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829689.991935:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829691.015864:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829691.015877:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829691.015899:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787829691.015916:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829692.039919:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829692.039919:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829692.039923:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829692.040036:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829692.936026:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829692.936034:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787829692.936038:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829692.936046:0:27606:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3128 00000100:00080000:0.0:1787829692.936056:0:27606:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829692.936079:0:27606:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787829692.936114:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7192c0 x1874672085887232/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829692.936162:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829692.936167:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829692.936170:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829692.936175:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3128 00000100:00080000:0.0:1787829692.936180:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829692.936186:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829692.936198:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829692.936224:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085887232:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1787829692.936233:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a1c0 x1874672085887360/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829692.936248:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829692.936252:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787829692.936255:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829692.936260:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3128 00000100:00080000:0.0:1787829692.936264:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829692.936267:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829692.936271:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829692.936282:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718b40 x1874672085887488/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787829692.936287:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787829692.936296:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829692.936299:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829692.936300:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829692.936320:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3128 00000100:00080000:0.0:1787829692.936323:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829692.936326:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829692.936330:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:0.0:1787829692.936338:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718000 x1874672085887616/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829692.936344:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829692.936347:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829692.936349:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829692.936352:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3128 00000100:00080000:0.0:1787829692.936356:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829692.936361:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787829692.936362:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829692/real 1787829692] req@ffff9e35da7192c0 x1874672085887232/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829717 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1787829692.936364:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:0.0:1787829692.936376:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a580 x1874672085887744/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829692.936384:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1787829692.936385:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7192c0 x1874672085887232/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829717 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829692.936391:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829692.936395:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829692.936419:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3128 00000100:00080000:3.0:1787829692.936419:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829692.936425:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829692.936427:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:0.0:1787829692.936432:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787829692.936436:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085887232:192.168.203.141@tcp:250:kworker.0 00000008:00100000:0.0:1787829692.936440:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829692.936467:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da719a40 x1874672085887872/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829692.936475:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1787829692.936478:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085887360:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787829692.936480:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787829692.936488:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829692.936492:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3128 00000100:00080000:0.0:1787829692.936496:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829692.936499:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829692.936503:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1787829692.936509:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829692.936513:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7183c0 x1874672085888000/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829692.936526:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1787829692.936527:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829692/real 1787829692] req@ffff9e35da71a1c0 x1874672085887360/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829717 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829692.936529:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829692.936532:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829692.936540:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3128 00000100:00080000:0.0:1787829692.936546:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829692.936551:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829692.936555:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:3.0:1787829692.936558:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a1c0 x1874672085887360/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829717 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829692.936565:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a940 x1874672085888128/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829692.936568:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829692.936577:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:0.0:1787829692.936582:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1787829692.936583:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085887360:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787829692.936588:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829692.936591:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1787829692.936594:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085887488:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787829692.936598:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3128 00000100:00080000:0.0:1787829692.936602:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829692.936623:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829692.936629:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000800:00000100:3.0:1787829692.936636:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829692.936646:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718f00 x1874672085888256/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787829692.936650:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829692/real 1787829692] req@ffff9e35da718b40 x1874672085887488/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829717 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829692.936708:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718b40 x1874672085887488/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829717 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829692.936714:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829692.936717:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787829692.936720:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085887488:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787829692.936724:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085887616:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787829692.936737:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787829692.936744:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829692/real 1787829692] req@ffff9e35da718000 x1874672085887616/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829717 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829692.936751:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718000 x1874672085887616/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829717 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829692.936756:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829692.936773:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787829692.936775:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085887616:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787829692.936779:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085887744:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787829692.936792:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787829692.936799:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829692/real 1787829692] req@ffff9e35da71a580 x1874672085887744/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829717 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829692.936807:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672085887744/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829717 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829692.936818:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829692.936820:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787829692.936839:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085887744:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787829692.936856:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085887872:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787829692.936920:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787829692.936928:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829692/real 1787829692] req@ffff9e35da719a40 x1874672085887872/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829717 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829692.936935:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672085887872/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829717 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829692.936941:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829692.936943:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787829692.936945:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085887872:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787829692.936950:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085888000:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787829692.936967:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787829692.936978:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829692/real 1787829692] req@ffff9e35da7183c0 x1874672085888000/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829717 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829692.937006:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672085888000/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829717 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829692.937016:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829692.937020:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787829692.937023:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085888000:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787829692.937028:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085888128:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787829692.937043:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787829692.937051:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829692/real 1787829692] req@ffff9e35da71a940 x1874672085888128/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829717 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829692.937075:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672085888128/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829717 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829692.937083:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829692.937085:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787829692.937087:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085888128:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787829692.937092:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085888256:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787829692.937104:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787829692.937112:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829692/real 1787829692] req@ffff9e35da718f00 x1874672085888256/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829717 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829692.937121:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672085888256/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829717 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829692.937127:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829692.937129:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787829692.937131:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085888256:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829693.063959:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829693.063969:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829693.064058:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829693.064145:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829694.087883:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829694.087905:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829694.087925:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829694.087953:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829695.111903:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829695.111907:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829695.111936:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829695.111962:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829696.135904:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829696.135905:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829696.135905:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829696.135953:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829697.159935:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829697.159958:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829697.159958:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:0.0:1787829697.160054:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829698.055928:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829698.055937:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829698.055940:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829698.055949:0:27606:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3133 00000100:00080000:2.0:1787829698.055958:0:27606:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829698.055964:0:27606:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787829698.055996:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498ed00 x1874672085888384/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829698.056020:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829698.056023:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829698.056025:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829698.056028:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3133 00000100:00080000:2.0:1787829698.056030:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829698.056033:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829698.056041:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829698.056048:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cf00 x1874672085888512/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829698.056060:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829698.056063:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829698.056065:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829698.056068:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3133 00000100:00080000:2.0:1787829698.056070:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829698.056073:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829698.056076:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829698.056083:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e580 x1874672085888640/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787829698.056088:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085888384:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1787829698.056094:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829698.056098:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829698.056099:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829698.056102:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3133 00000100:00080000:2.0:1787829698.056105:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829698.056108:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829698.056113:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:2.0:1787829698.056127:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f480 x1874672085888768/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829698.056138:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829698.056142:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829698.056144:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829698.056151:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3133 00000800:00000100:3.0:1787829698.056152:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1787829698.056160:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829698.056164:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829698.056169:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:2.0:1787829698.056181:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e1c0 x1874672085888896/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829698.056188:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829698.056192:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:3.0:1787829698.056192:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829698/real 1787829698] req@ffff9e35c498ed00 x1874672085888384/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829723 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829698.056196:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829698.056204:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3133 00000100:00080000:2.0:1787829698.056209:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829698.056212:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829698.056215:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829698.056217:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498ed00 x1874672085888384/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829723 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829698.056226:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c780 x1874672085889024/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829698.056227:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829698.056235:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:2.0:1787829698.056240:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1787829698.056243:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085888384:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1787829698.056244:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829698.056248:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829698.056253:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3133 00000100:00100000:3.0:1787829698.056255:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085888512:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1787829698.056256:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829698.056263:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829698.056266:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829698.056275:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c3c0 x1874672085889152/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787829698.056283:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1787829698.056292:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829698.056296:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:3.0:1787829698.056297:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829698/real 1787829698] req@ffff9e35c498cf00 x1874672085888512/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829723 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829698.056298:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829698.056306:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3133 00000100:00080000:2.0:1787829698.056312:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829698.056318:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787829698.056319:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498cf00 x1874672085888512/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829723 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1787829698.056324:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:2.0:1787829698.056338:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f840 x1874672085889280/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829698.056338:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829698.056351:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:2.0:1787829698.056352:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829698.056357:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829698.056361:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1787829698.056361:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085888512:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1787829698.056379:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3133 00000100:00080000:2.0:1787829698.056383:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829698.056386:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829698.056389:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:2.0:1787829698.056421:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498de00 x1874672085889408/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787829698.056426:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085888640:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787829698.056460:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787829698.056472:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829698/real 1787829698] req@ffff9e35c498e580 x1874672085888640/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829723 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829698.056481:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e580 x1874672085888640/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829723 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829698.056487:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829698.056489:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787829698.056491:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085888640:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787829698.056495:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085888768:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787829698.056507:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787829698.056513:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829698/real 1787829698] req@ffff9e35c498f480 x1874672085888768/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829723 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829698.056521:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f480 x1874672085888768/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829723 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829698.056526:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829698.056528:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787829698.056530:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085888768:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787829698.056534:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085888896:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787829698.056562:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787829698.056568:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829698/real 1787829698] req@ffff9e35c498e1c0 x1874672085888896/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829723 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829698.056575:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e1c0 x1874672085888896/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829723 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829698.056581:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829698.056583:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787829698.056585:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085888896:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787829698.056588:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085889024:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787829698.056597:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787829698.056604:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829698/real 1787829698] req@ffff9e35c498c780 x1874672085889024/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829723 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829698.056610:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c780 x1874672085889024/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829723 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829698.056616:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829698.056618:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787829698.056619:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085889024:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787829698.056623:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085889152:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787829698.056642:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787829698.056649:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829698/real 1787829698] req@ffff9e35c498c3c0 x1874672085889152/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829723 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829698.056657:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c3c0 x1874672085889152/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829723 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829698.056662:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829698.056664:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787829698.056667:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085889152:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787829698.056671:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085889280:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787829698.056681:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787829698.056686:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829698/real 1787829698] req@ffff9e35c498f840 x1874672085889280/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829723 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829698.056693:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f840 x1874672085889280/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829723 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829698.056699:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829698.056700:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787829698.056702:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085889280:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787829698.056726:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085889408:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787829698.056738:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787829698.056745:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829698/real 1787829698] req@ffff9e35c498de00 x1874672085889408/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829723 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829698.056752:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498de00 x1874672085889408/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829723 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829698.056757:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829698.056759:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787829698.056761:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085889408:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829698.183929:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829698.183929:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829698.183958:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829698.183975:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829699.207862:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829699.207863:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787829699.207892:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829699.207953:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829700.231875:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829700.231876:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787829700.231881:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:0.0:1787829700.231984:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829701.255914:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829701.255969:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829701.255981:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829701.255985:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829702.279897:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829702.279929:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829702.279938:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787829702.279989:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829703.175930:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829703.175937:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787829703.175941:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829703.175949:0:27606:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3138 00000100:00080000:0.0:1787829703.175958:0:27606:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829703.175964:0:27606:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787829703.175997:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718780 x1874672085889536/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829703.176023:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829703.176030:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829703.176032:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829703.176035:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3138 00000100:00080000:0.0:1787829703.176037:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829703.176039:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829703.176049:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829703.176066:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718f00 x1874672085889664/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787829703.176072:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085889536:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1787829703.176086:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829703.176090:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787829703.176094:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829703.176099:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3138 00000100:00080000:0.0:1787829703.176104:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829703.176109:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829703.176115:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829703.176126:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a940 x1874672085889792/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829703.176133:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:3.0:1787829703.176135:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787829703.176137:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829703.176140:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829703.176144:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3138 00000100:00080000:0.0:1787829703.176147:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829703.176151:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829703.176156:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:0.0:1787829703.176165:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7183c0 x1874672085889920/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829703.176171:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829703.176174:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829703.176176:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1787829703.176179:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829703/real 1787829703] req@ffff9e35da718780 x1874672085889536/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829728 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829703.176182:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3138 00000100:00080000:0.0:1787829703.176187:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829703.176193:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829703.176199:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:3.0:1787829703.176207:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718780 x1874672085889536/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829728 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829703.176210:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da719a40 x1874672085890048/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829703.176235:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829703.176235:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829703.176241:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829703.176243:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:0.0:1787829703.176244:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829703.176252:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3138 00000100:00100000:3.0:1787829703.176252:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085889536:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1787829703.176259:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829703.176263:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829703.176266:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829703.176272:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a580 x1874672085890176/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829703.176280:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1787829703.176281:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085889664:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787829703.176295:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787829703.176299:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829703.176304:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3138 00000800:00000100:3.0:1787829703.176305:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787829703.176309:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829703.176313:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829703.176317:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829703.176327:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718000 x1874672085890304/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787829703.176342:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829703/real 1787829703] req@ffff9e35da718f00 x1874672085889664/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829728 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829703.176377:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1787829703.176379:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672085889664/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829728 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829703.176382:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829703.176386:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829703.176394:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3138 00000100:00080000:3.0:1787829703.176395:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829703.176401:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829703.176403:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:0.0:1787829703.176407:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787829703.176410:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085889664:192.168.203.141@tcp:38:kworker.0 00000008:00100000:0.0:1787829703.176414:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:3.0:1787829703.176418:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085889792:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829703.176426:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718b40 x1874672085890432/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829703.176434:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829703.176436:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829703.176438:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:3.0:1787829703.176442:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787829703.176446:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3138 00000100:00080000:0.0:1787829703.176450:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829703.176453:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829703.176456:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:3.0:1787829703.176464:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829703/real 1787829703] req@ffff9e35da71a940 x1874672085889792/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829728 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829703.176467:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a1c0 x1874672085890560/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787829703.176497:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672085889792/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829728 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829703.176505:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829703.176509:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787829703.176511:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085889792:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787829703.176516:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085889920:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787829703.176529:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787829703.176536:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829703/real 1787829703] req@ffff9e35da7183c0 x1874672085889920/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829728 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829703.176544:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672085889920/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829728 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829703.176549:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829703.176551:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787829703.176553:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085889920:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787829703.176556:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085890048:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787829703.176566:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787829703.176572:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829703/real 1787829703] req@ffff9e35da719a40 x1874672085890048/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829728 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829703.176584:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672085890048/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829728 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829703.176590:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829703.176592:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787829703.176593:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085890048:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787829703.176606:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085890176:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787829703.176617:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787829703.176624:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829703/real 1787829703] req@ffff9e35da71a580 x1874672085890176/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829728 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829703.176632:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672085890176/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829728 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829703.176637:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829703.176639:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787829703.176640:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085890176:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787829703.176644:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085890304:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787829703.176654:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787829703.176660:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829703/real 1787829703] req@ffff9e35da718000 x1874672085890304/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829728 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829703.176667:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718000 x1874672085890304/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829728 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829703.176676:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829703.176678:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787829703.176680:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085890304:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787829703.176683:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085890432:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787829703.176693:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787829703.176698:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829703/real 1787829703] req@ffff9e35da718b40 x1874672085890432/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829728 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829703.176705:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718b40 x1874672085890432/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829728 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829703.176710:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829703.176712:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787829703.176714:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085890432:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787829703.176718:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085890560:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787829703.176727:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787829703.176733:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829703/real 1787829703] req@ffff9e35da71a1c0 x1874672085890560/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829728 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829703.176740:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a1c0 x1874672085890560/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829728 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829703.176745:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829703.176746:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787829703.176749:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085890560:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829703.303939:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829703.303955:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829703.304017:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829703.304041:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829704.327896:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829704.327896:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829704.327929:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829704.327969:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829705.351923:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829705.351923:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829705.351986:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829705.351994:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829706.375874:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829706.375878:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829706.375901:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829706.375938:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829707.399977:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829707.399977:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829707.400049:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829707.400058:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787829708.295902:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829708.295913:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829708.295918:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829708.295929:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3143 00000100:00080000:1.0:1787829708.295940:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829708.295948:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787829708.295988:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f43c0 x1874672085890688/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829708.296016:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829708.296020:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829708.296022:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829708.296026:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3143 00000100:00080000:1.0:1787829708.296031:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829708.296036:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829708.296048:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829708.296068:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085890688:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1787829708.296069:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5a40 x1874672085890816/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829708.296086:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829708.296090:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829708.296092:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829708.296097:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3143 00000100:00080000:1.0:1787829708.296101:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829708.296105:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829708.296110:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829708.296124:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5e00 x1874672085890944/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829708.296134:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829708.296137:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829708.296139:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:3.0:1787829708.296145:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787829708.296148:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3143 00000100:00080000:1.0:1787829708.296155:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829708.296160:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829708.296165:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:1.0:1787829708.296179:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4b40 x1874672085891072/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787829708.296190:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829708/real 1787829708] req@ffff9e35da6f43c0 x1874672085890688/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829733 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829708.296192:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829708.296202:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829708.296206:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829708.296214:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3143 00000100:00100000:3.0:1787829708.296219:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f43c0 x1874672085890688/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829733 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829708.296220:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829708.296229:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:3.0:1787829708.296240:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:1.0:1787829708.296241:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00080000:3.0:1787829708.296247:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787829708.296251:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085890688:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1787829708.296260:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f70c0 x1874672085891200/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829708.296272:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829708.296275:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:3.0:1787829708.296278:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085890816:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787829708.296279:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829708.296288:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3143 00000100:00080000:1.0:1787829708.296292:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829708.296309:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829708.296314:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1787829708.296326:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829708.296327:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f43c0 x1874672085891328/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829708.296337:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829708.296339:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829708.296341:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829708.296343:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3143 00000100:00080000:1.0:1787829708.296347:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829708.296352:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787829708.296353:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829708/real 1787829708] req@ffff9e35da6f5a40 x1874672085890816/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829733 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1787829708.296357:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829708.296369:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7840 x1874672085891456/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787829708.296377:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5a40 x1874672085890816/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829733 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829708.296378:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829708.296384:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829708.296387:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829708.296396:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3143 00000100:00080000:1.0:1787829708.296401:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829708.296403:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829708.296406:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:3.0:1787829708.296411:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000008:00100000:1.0:1787829708.296412:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:3.0:1787829708.296421:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085890816:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829708.296423:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f61c0 x1874672085891584/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829708.296434:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1787829708.296434:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085890944:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787829708.296441:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829708.296444:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829708.296447:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3143 00000100:00080000:1.0:1787829708.296450:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829708.296452:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829708.296456:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000800:00000100:3.0:1787829708.296466:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829708.296467:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4f00 x1874672085891712/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787829708.296486:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829708/real 1787829708] req@ffff9e35da6f5e00 x1874672085890944/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829733 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829708.296499:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5e00 x1874672085890944/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829733 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829708.296508:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829708.296516:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787829708.296519:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085890944:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787829708.296526:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085891072:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787829708.296544:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787829708.296555:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829708/real 1787829708] req@ffff9e35da6f4b40 x1874672085891072/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829733 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829708.296568:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4b40 x1874672085891072/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829733 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829708.296577:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829708.296580:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787829708.296584:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085891072:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787829708.296590:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085891200:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787829708.296611:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787829708.296624:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829708/real 1787829708] req@ffff9e35da6f70c0 x1874672085891200/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829733 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829708.296638:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f70c0 x1874672085891200/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829733 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829708.296648:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829708.296651:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787829708.296654:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085891200:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787829708.296675:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085891328:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787829708.296693:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787829708.296702:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829708/real 1787829708] req@ffff9e35da6f43c0 x1874672085891328/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829733 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829708.296716:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f43c0 x1874672085891328/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829733 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829708.296724:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829708.296726:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787829708.296730:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085891328:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787829708.296737:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085891456:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787829708.296753:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787829708.296762:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829708/real 1787829708] req@ffff9e35da6f7840 x1874672085891456/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829733 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829708.296774:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7840 x1874672085891456/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829733 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829708.296783:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829708.296787:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787829708.296790:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085891456:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787829708.296927:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085891584:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787829708.296949:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787829708.296961:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829708/real 1787829708] req@ffff9e35da6f61c0 x1874672085891584/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829733 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829708.296974:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f61c0 x1874672085891584/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829733 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829708.296980:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829708.296983:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787829708.296985:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085891584:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787829708.296990:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085891712:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787829708.297006:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787829708.297015:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829708/real 1787829708] req@ffff9e35da6f4f00 x1874672085891712/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829733 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829708.297028:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4f00 x1874672085891712/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829733 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829708.297037:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829708.297040:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787829708.297044:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085891712:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829708.423896:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829708.423914:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829708.423919:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829708.423927:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829709.447970:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829709.447984:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829709.448018:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829709.448075:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829710.471912:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829710.471942:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829710.471993:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829710.471995:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829711.495918:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829711.495935:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829711.495968:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829711.495972:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829712.519934:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829712.519934:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829712.519976:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829712.520076:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00080000:0.0:1787829713.415895:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829713.415912:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787829713.415917:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829713.415929:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3148 00000100:00080000:0.0:1787829713.415940:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829713.415948:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787829713.415991:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7192c0 x1874672085891840/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829713.416032:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829713.416036:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829713.416037:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829713.416041:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3148 00000100:00080000:0.0:1787829713.416046:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829713.416051:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829713.416064:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829713.416085:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a1c0 x1874672085891968/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829713.416093:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085891840:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1787829713.416101:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829713.416105:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787829713.416107:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829713.416111:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3148 00000100:00080000:0.0:1787829713.416116:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829713.416120:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829713.416125:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829713.416138:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718b40 x1874672085892096/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829713.416151:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829713.416154:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829713.416156:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829713.416159:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3148 00000100:00080000:0.0:1787829713.416170:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829713.416173:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829713.416177:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000800:00000100:2.0:1787829713.416183:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829713.416190:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718000 x1874672085892224/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829713.416198:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829713.416201:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829713.416202:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829713.416207:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3148 00000100:00080000:0.0:1787829713.416209:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829713.416212:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829713.416215:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:0.0:1787829713.416224:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a580 x1874672085892352/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829713.416232:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829713.416237:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:2.0:1787829713.416237:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829713/real 1787829713] req@ffff9e35da7192c0 x1874672085891840/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829738 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829713.416240:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829713.416248:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3148 00000100:00080000:0.0:1787829713.416255:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829713.416275:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829713.416279:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829713.416287:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7192c0 x1874672085891840/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829738 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829713.416291:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da719a40 x1874672085892480/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829713.416316:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829713.416320:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829713.416321:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829713.416322:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829713.416329:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3148 00000100:00080000:0.0:1787829713.416334:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829713.416334:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:0.0:1787829713.416342:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787829713.416346:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085891840:192.168.203.141@tcp:250:kworker.0 00000008:00100000:0.0:1787829713.416348:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829713.416369:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7183c0 x1874672085892608/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829713.416377:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829713.416382:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:2.0:1787829713.416382:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085891968:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787829713.416385:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829713.416391:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3148 00000100:00080000:0.0:1787829713.416395:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829713.416398:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829713.416401:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000800:00000100:2.0:1787829713.416415:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829713.416416:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a940 x1874672085892736/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829713.416430:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829713.416432:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829713.416435:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787829713.416437:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829713/real 1787829713] req@ffff9e35da71a1c0 x1874672085891968/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829738 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829713.416440:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3148 00000100:00080000:0.0:1787829713.416445:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829713.416452:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829713.416459:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:2.0:1787829713.416468:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a1c0 x1874672085891968/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829738 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829713.416471:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718f00 x1874672085892864/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829713.416510:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829713.416516:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829713.416522:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085891968:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829713.416529:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085892096:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787829713.416551:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829713.416560:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829713/real 1787829713] req@ffff9e35da718b40 x1874672085892096/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829738 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829713.416568:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718b40 x1874672085892096/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829738 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829713.416574:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829713.416577:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829713.416579:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085892096:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829713.416583:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085892224:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829713.416593:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829713.416599:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829713/real 1787829713] req@ffff9e35da718000 x1874672085892224/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829738 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829713.416606:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718000 x1874672085892224/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829738 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829713.416612:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829713.416614:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829713.416617:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085892224:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829713.416621:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085892352:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829713.416632:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829713.416637:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829713/real 1787829713] req@ffff9e35da71a580 x1874672085892352/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829738 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829713.416644:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672085892352/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829738 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829713.416651:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829713.416654:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829713.416657:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085892352:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829713.416663:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085892480:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787829713.416682:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829713.416694:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829713/real 1787829713] req@ffff9e35da719a40 x1874672085892480/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829738 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829713.416706:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672085892480/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829738 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829713.416713:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829713.416715:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829713.416717:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085892480:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829713.416742:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085892608:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787829713.416758:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829713.416766:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829713/real 1787829713] req@ffff9e35da7183c0 x1874672085892608/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829738 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829713.416779:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672085892608/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829738 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829713.416788:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829713.416792:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829713.416796:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085892608:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829713.416833:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085892736:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829713.416853:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829713.416861:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829713/real 1787829713] req@ffff9e35da71a940 x1874672085892736/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829738 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829713.416870:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672085892736/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829738 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829713.416876:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829713.416878:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829713.416880:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085892736:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829713.416884:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085892864:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829713.416893:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829713.416899:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829713/real 1787829713] req@ffff9e35da718f00 x1874672085892864/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829738 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829713.416906:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672085892864/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829738 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829713.416912:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829713.416914:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829713.416916:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085892864:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829713.543878:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829713.543884:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829713.543918:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829713.543940:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829714.567926:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829714.567926:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829714.567939:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:0.0:1787829714.567980:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829715.591915:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829715.591919:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829715.591932:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829715.591966:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829716.615912:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829716.615913:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787829716.615913:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829716.615993:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829717.639863:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829717.639876:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829717.639909:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:0.0:1787829717.639932:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829718.535887:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829718.535897:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787829718.535901:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829718.535912:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3153 00000100:00080000:0.0:1787829718.535922:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829718.535930:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787829718.535965:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718780 x1874672085892992/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829718.535991:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829718.535995:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829718.535998:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829718.536002:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3153 00000100:00080000:0.0:1787829718.536005:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829718.536010:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829718.536022:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829718.536029:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085892992:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1787829718.536037:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718f00 x1874672085893120/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829718.536051:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829718.536054:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787829718.536056:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829718.536060:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3153 00000100:00080000:0.0:1787829718.536063:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829718.536067:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829718.536072:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829718.536084:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a940 x1874672085893248/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829718.536094:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829718.536098:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829718.536100:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829718.536104:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3153 00000100:00080000:0.0:1787829718.536107:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829718.536116:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829718.536122:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:0.0:1787829718.536134:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7183c0 x1874672085893376/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829718.536138:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085893120:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787829718.536155:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829718.536159:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829718.536162:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829718.536168:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3153 00000100:00080000:0.0:1787829718.536173:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787829718.536176:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085893248:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787829718.536180:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829718.536187:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:0.0:1787829718.536202:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da719a40 x1874672085893504/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829718.536205:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085893376:192.168.203.141@tcp:8:kworker.0 00000100:00080000:0.0:1787829718.536217:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829718.536221:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829718.536239:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829718.536244:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3153 00000100:00080000:0.0:1787829718.536249:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829718.536253:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829718.536259:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829718.536263:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085893504:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829718.536275:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a580 x1874672085893632/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829718.536311:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829718.536315:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787829718.536317:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829718.536326:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3153 00000100:00100000:2.0:1787829718.536327:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085893632:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787829718.536330:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829718.536337:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829718.536343:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829718.536356:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718000 x1874672085893760/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829718.536374:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829718.536378:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829718.536381:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829718.536386:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3153 00000100:00080000:0.0:1787829718.536390:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787829718.536391:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085893760:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787829718.536398:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829718.536404:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:0.0:1787829718.536418:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718b40 x1874672085893888/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829718.536431:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829718.536435:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829718.536437:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829718.536443:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3153 00000100:00100000:2.0:1787829718.536444:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085893888:192.168.203.141@tcp:8:kworker.0 00000100:00080000:0.0:1787829718.536448:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829718.536454:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829718.536459:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:0.0:1787829718.536473:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a1c0 x1874672085894016/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829718.536502:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085894016:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829718.663895:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829718.663895:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829718.663931:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829718.663949:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829719.687898:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829719.687924:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829719.687954:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829719.688006:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829720.711906:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829720.711922:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829720.711932:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:1.0:1787829720.711961:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1787829721.608065:0:2399:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.141:988 00000400:00000100:3.0:1787829721.608110:0:2399:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.141@tcp at host 192.168.203.141 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1787829721.608143:0:2399:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1787829721.608196:0:2399:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000800:00000100:3.0:1787829721.608224:0:2399:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1787829721.608231:0:2399:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1787829721.608239:0:2399:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1787829721.608245:0:2399:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1787829721.608253:0:2399:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1787829721.608259:0:2399:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1787829721.608267:0:2399:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:2.0:1787829721.608270:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829718/real 1787829721] req@ffff9e35da71a1c0 x1874672085894016/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829743 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829721.608318:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a1c0 x1874672085894016/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829743 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829721.608332:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829721.608337:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829721.608342:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085894016:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829721.608356:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829718/real 1787829721] req@ffff9e35da718b40 x1874672085893888/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829743 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829721.608371:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718b40 x1874672085893888/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829743 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829721.608378:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829721.608380:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829721.608383:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085893888:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829721.608387:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829718/real 1787829721] req@ffff9e35da718000 x1874672085893760/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829743 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829721.608400:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718000 x1874672085893760/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829743 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829721.608408:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829721.608412:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829721.608415:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085893760:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829721.608422:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829718/real 1787829721] req@ffff9e35da71a580 x1874672085893632/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829743 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829721.608434:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672085893632/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829743 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829721.608442:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829721.608446:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829721.608449:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085893632:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829721.608455:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829718/real 1787829721] req@ffff9e35da719a40 x1874672085893504/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829743 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829721.608465:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672085893504/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829743 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829721.608473:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829721.608476:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829721.608479:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085893504:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829721.608484:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829718/real 1787829721] req@ffff9e35da7183c0 x1874672085893376/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829743 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829721.608495:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672085893376/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829743 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829721.608502:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829721.608505:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829721.608508:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085893376:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829721.608514:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829718/real 1787829721] req@ffff9e35da718f00 x1874672085893120/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829743 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829721.608526:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672085893120/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829743 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829721.608533:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829721.608536:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829721.608539:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085893120:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829721.608545:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829718/real 1787829721] req@ffff9e35da71a940 x1874672085893248/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829743 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829721.608555:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672085893248/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829743 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829721.608562:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829721.608565:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829721.608568:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085893248:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829721.608573:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829718/real 1787829721] req@ffff9e35da718780 x1874672085892992/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829743 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829721.608584:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718780 x1874672085892992/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829743 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829721.608592:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829721.608596:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829721.608599:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085892992:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1787829721.735914:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829721.735918:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829721.735933:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829721.735949:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829722.759931:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829722.759937:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829722.759972:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829722.759988:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829723.655931:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829723.655942:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829723.655946:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829723.655957:0:27606:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3159 00000100:00080000:1.0:1787829723.655968:0:27606:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829723.655976:0:27606:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787829723.656013:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7480 x1874672085894144/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829723.656045:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829723.656049:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829723.656052:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829723.656056:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3159 00000100:00080000:1.0:1787829723.656059:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829723.656068:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829723.656081:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829723.656096:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4000 x1874672085894272/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829723.656107:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829723.656111:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829723.656113:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829723.656117:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3159 00000100:00080000:1.0:1787829723.656121:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829723.656127:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829723.656133:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829723.656150:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6940 x1874672085894400/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829723.656153:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085894144:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787829723.656171:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829723.656175:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829723.656177:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829723.656181:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3159 00000100:00080000:1.0:1787829723.656185:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829723.656205:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829723.656211:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:1.0:1787829723.656225:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6d00 x1874672085894528/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829723.656236:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829723.656241:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829723.656244:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829723.656248:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3159 00000100:00080000:1.0:1787829723.656254:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1787829723.656259:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787829723.656263:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829723.656271:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:1.0:1787829723.656285:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4780 x1874672085894656/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829723.656306:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787829723.656311:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829723/real 1787829723] req@ffff9e35da6f7480 x1874672085894144/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829748 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829723.656312:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829723.656316:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829723.656326:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3159 00000100:00080000:1.0:1787829723.656333:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829723.656338:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787829723.656344:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7480 x1874672085894144/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829748 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1787829723.656345:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1787829723.656361:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1787829723.656364:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6580 x1874672085894784/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829723.656369:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829723.656377:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085894144:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787829723.656387:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787829723.656391:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085894272:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787829723.656394:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829723.656399:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829723.656405:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3159 00000100:00080000:1.0:1787829723.656410:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829723.656415:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829723.656422:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1787829723.656428:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829723.656440:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4f00 x1874672085894912/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829723.656447:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829723/real 1787829723] req@ffff9e35da6f4000 x1874672085894272/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829748 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829723.656455:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829723.656466:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829723.656469:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787829723.656475:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4000 x1874672085894272/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829748 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829723.656478:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3159 00000100:00080000:1.0:1787829723.656485:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829723.656493:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:2.0:1787829723.656494:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000008:00100000:1.0:1787829723.656500:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00080000:2.0:1787829723.656502:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829723.656509:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085894272:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829723.656516:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f61c0 x1874672085895040/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829723.656528:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829723.656532:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829723.656534:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829723.656538:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3159 00000100:00080000:1.0:1787829723.656542:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787829723.656548:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085894400:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787829723.656550:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829723.656560:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:1.0:1787829723.656583:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7840 x1874672085895168/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829723.656592:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829723.656613:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829723/real 1787829723] req@ffff9e35da6f6940 x1874672085894400/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829748 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829723.656628:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6940 x1874672085894400/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829748 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829723.656638:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829723.656641:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829723.656645:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085894400:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829723.656651:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085894528:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829723.656670:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829723.656680:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829723/real 1787829723] req@ffff9e35da6f6d00 x1874672085894528/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829748 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829723.656690:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6d00 x1874672085894528/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829748 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829723.656698:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829723.656701:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829723.656704:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085894528:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829723.656710:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085894656:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829723.656727:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829723.656737:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829723/real 1787829723] req@ffff9e35da6f4780 x1874672085894656/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829748 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829723.656755:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4780 x1874672085894656/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829748 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829723.656764:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829723.656768:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829723.656771:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085894656:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829723.656777:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085894784:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787829723.656826:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829723.656840:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829723/real 1787829723] req@ffff9e35da6f6580 x1874672085894784/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829748 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829723.656854:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6580 x1874672085894784/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829748 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829723.656862:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829723.656866:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829723.656869:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085894784:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829723.656875:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085894912:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787829723.656896:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829723.656907:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829723/real 1787829723] req@ffff9e35da6f4f00 x1874672085894912/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829748 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829723.656919:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4f00 x1874672085894912/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829748 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829723.656934:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829723.656938:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829723.656941:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085894912:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829723.656948:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085895040:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829723.656969:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829723.656981:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829723/real 1787829723] req@ffff9e35da6f61c0 x1874672085895040/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829748 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829723.656992:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f61c0 x1874672085895040/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829748 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829723.657001:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829723.657005:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829723.657008:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085895040:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829723.657037:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085895168:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829723.657057:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829723.657068:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829723/real 1787829723] req@ffff9e35da6f7840 x1874672085895168/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829748 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829723.657081:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7840 x1874672085895168/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829748 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829723.657088:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829723.657092:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829723.657095:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085895168:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829723.783882:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829723.783892:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829723.783920:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787829723.783946:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829724.807904:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829724.807904:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829724.807906:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787829724.807950:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829725.831879:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829725.831891:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829725.831900:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:1.0:1787829725.831934:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829726.855880:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829726.855883:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829726.855885:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787829726.855925:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829727.879891:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829727.879917:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829727.879927:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787829727.879941:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829728.775906:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829728.775916:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787829728.775920:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829728.775931:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3164 00000100:00080000:0.0:1787829728.775942:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829728.775950:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787829728.775987:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7192c0 x1874672085895296/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829728.776015:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829728.776019:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829728.776021:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829728.776025:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3164 00000100:00080000:0.0:1787829728.776029:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829728.776034:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829728.776045:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829728.776055:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085895296:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1787829728.776064:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718780 x1874672085895424/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829728.776078:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829728.776082:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787829728.776084:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829728.776088:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3164 00000100:00080000:0.0:1787829728.776093:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829728.776097:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829728.776103:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829728.776119:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a940 x1874672085895552/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829728.776124:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787829728.776136:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829728.776141:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829728.776143:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829728.776149:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3164 00000100:00080000:0.0:1787829728.776169:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829728.776174:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829728.776181:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:2.0:1787829728.776186:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829728/real 1787829728] req@ffff9e35da7192c0 x1874672085895296/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829753 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829728.776198:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718f00 x1874672085895680/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829728.776212:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7192c0 x1874672085895296/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829753 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829728.776213:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829728.776222:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829728.776227:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829728.776236:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829728.776238:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3164 00000100:00080000:2.0:1787829728.776245:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:0.0:1787829728.776246:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787829728.776252:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085895296:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1787829728.776255:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829728.776264:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:0.0:1787829728.776277:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7183c0 x1874672085895808/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829728.776283:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085895424:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787829728.776300:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829728.776304:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829728.776307:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829728.776313:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3164 00000800:00000100:2.0:1787829728.776318:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787829728.776320:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829728.776328:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829728.776334:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829728.776336:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829728/real 1787829728] req@ffff9e35da718780 x1874672085895424/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829753 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829728.776350:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da719a40 x1874672085895936/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829728.776356:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718780 x1874672085895424/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829753 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829728.776376:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829728.776379:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829728.776382:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787829728.776387:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829728.776388:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:0.0:1787829728.776395:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3164 00000100:00100000:2.0:1787829728.776399:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085895424:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787829728.776402:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787829728.776411:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085895552:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787829728.776412:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829728.776422:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829728.776438:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a580 x1874672085896064/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829728.776444:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829728.776460:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829728/real 1787829728] req@ffff9e35da71a940 x1874672085895552/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829753 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829728.776464:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829728.776474:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829728.776477:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787829728.776485:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672085895552/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829753 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829728.776486:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3164 00000100:00080000:0.0:1787829728.776495:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829728.776502:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829728.776504:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:2.0:1787829728.776509:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000008:00100000:0.0:1787829728.776514:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:2.0:1787829728.776516:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085895552:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829728.776526:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085895680:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829728.776533:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718000 x1874672085896192/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829728.776546:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1787829728.776551:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787829728.776553:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829728.776557:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829728.776563:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3164 00000100:00100000:2.0:1787829728.776568:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829728/real 1787829728] req@ffff9e35da718f00 x1874672085895680/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829753 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829728.776570:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829728.776579:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829728.776589:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:2.0:1787829728.776592:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672085895680/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829753 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829728.776603:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1787829728.776607:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718b40 x1874672085896320/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829728.776610:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829728.776618:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085895680:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829728.776646:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085895808:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829728.776666:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829728.776677:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829728/real 1787829728] req@ffff9e35da7183c0 x1874672085895808/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829753 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829728.776690:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672085895808/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829753 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829728.776699:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829728.776702:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829728.776706:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085895808:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829728.776712:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085895936:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787829728.776733:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829728.776744:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829728/real 1787829728] req@ffff9e35da719a40 x1874672085895936/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829753 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829728.776757:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672085895936/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829753 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829728.776766:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829728.776769:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829728.776773:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085895936:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829728.776779:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085896064:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787829728.776826:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829728.776839:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829728/real 1787829728] req@ffff9e35da71a580 x1874672085896064/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829753 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829728.776859:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672085896064/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829753 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829728.776868:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829728.776871:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829728.776875:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085896064:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829728.776881:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085896192:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829728.776900:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829728.776909:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829728/real 1787829728] req@ffff9e35da718000 x1874672085896192/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829753 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829728.776922:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718000 x1874672085896192/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829753 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829728.776931:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829728.776934:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829728.776938:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085896192:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829728.776944:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085896320:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829728.776969:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829728.776980:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829728/real 1787829728] req@ffff9e35da718b40 x1874672085896320/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829753 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829728.776992:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718b40 x1874672085896320/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829753 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829728.777007:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829728.777010:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829728.777014:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085896320:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829728.903887:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829728.903889:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829728.903896:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787829728.903964:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829729.927866:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829729.927963:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829729.927978:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787829729.928018:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829730.951896:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829730.951896:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829730.951899:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829730.951945:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829731.975902:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829731.975902:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829731.975928:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:0.0:1787829731.975964:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829732.999911:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829732.999917:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787829732.999972:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829732.999973:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00080000:1.0:1787829733.895917:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829733.895925:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829733.895929:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829733.895943:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3169 00000100:00080000:1.0:1787829733.895953:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829733.895960:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787829733.895994:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f43c0 x1874672085896448/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829733.896016:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829733.896019:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829733.896021:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829733.896024:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3169 00000100:00080000:1.0:1787829733.896026:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829733.896028:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829733.896038:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829733.896047:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f70c0 x1874672085896576/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829733.896056:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829733.896058:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829733.896059:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829733.896062:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3169 00000100:00080000:1.0:1787829733.896064:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829733.896066:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829733.896070:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829733.896088:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4b40 x1874672085896704/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829733.896089:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085896448:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787829733.896129:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829733.896132:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829733.896133:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829733.896137:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3169 00000100:00080000:1.0:1787829733.896140:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829733.896144:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829733.896148:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:1.0:1787829733.896158:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5e00 x1874672085896832/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829733.896165:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829733.896168:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829733.896170:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829733.896173:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3169 00000100:00080000:1.0:1787829733.896177:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829733.896179:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829733.896182:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:1.0:1787829733.896189:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5a40 x1874672085896960/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829733.896196:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1787829733.896204:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787829733.896205:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829733.896208:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829733.896212:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3169 00000100:00080000:1.0:1787829733.896217:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829733.896220:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829733.896227:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829733.896237:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7840 x1874672085897088/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829733.896243:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829733.896246:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829733.896248:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829733.896261:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3169 00000100:00100000:2.0:1787829733.896263:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829733/real 1787829733] req@ffff9e35da6f43c0 x1874672085896448/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829758 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829733.896265:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829733.896276:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829733.896291:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829733.896302:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f43c0 x1874672085896448/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829758 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829733.896314:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f61c0 x1874672085897216/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829733.896316:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829733.896326:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829733.896339:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085896448:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1787829733.896355:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085896576:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787829733.896364:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829733.896369:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829733.896372:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829733.896379:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3169 00000800:00000100:2.0:1787829733.896381:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787829733.896393:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787829733.896400:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829733/real 1787829733] req@ffff9e35da6f70c0 x1874672085896576/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829758 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829733.896404:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829733.896421:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:2.0:1787829733.896441:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f70c0 x1874672085896576/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829758 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829733.896450:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4f00 x1874672085897344/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829733.896455:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829733.896468:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829733.896481:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085896576:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787829733.896488:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829733.896493:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829733.896495:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829733.896501:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3169 00000100:00080000:1.0:1787829733.896506:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829733.896513:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787829733.896518:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085896704:192.168.203.141@tcp:38:kworker.0 00000008:00100000:1.0:1787829733.896519:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:1.0:1787829733.896538:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6580 x1874672085897472/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829733.896568:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829733.896584:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829733/real 1787829733] req@ffff9e35da6f4b40 x1874672085896704/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829758 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829733.896594:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4b40 x1874672085896704/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829758 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829733.896601:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829733.896604:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829733.896606:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085896704:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829733.896611:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085896832:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829733.896629:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829733.896636:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829733/real 1787829733] req@ffff9e35da6f5e00 x1874672085896832/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829758 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829733.896643:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5e00 x1874672085896832/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829758 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829733.896654:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829733.896657:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829733.896666:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085896832:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829733.896671:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085896960:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829733.896684:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829733.896690:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829733/real 1787829733] req@ffff9e35da6f5a40 x1874672085896960/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829758 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829733.896698:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5a40 x1874672085896960/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829758 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829733.896704:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829733.896706:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829733.896708:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085896960:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829733.896712:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085897088:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787829733.896723:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829733.896729:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829733/real 1787829733] req@ffff9e35da6f7840 x1874672085897088/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829758 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829733.896735:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7840 x1874672085897088/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829758 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829733.896741:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829733.896743:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829733.896746:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085897088:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829733.896751:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085897216:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787829733.896761:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829733.896773:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829733/real 1787829733] req@ffff9e35da6f61c0 x1874672085897216/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829758 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829733.896780:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f61c0 x1874672085897216/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829758 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829733.896786:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829733.896788:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829733.896790:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085897216:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829733.896794:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085897344:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829733.896850:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829733.896858:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829733/real 1787829733] req@ffff9e35da6f4f00 x1874672085897344/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829758 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829733.896865:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4f00 x1874672085897344/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829758 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829733.896872:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829733.896874:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829733.896876:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085897344:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829733.896896:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085897472:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829733.896907:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829733.896922:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829733/real 1787829733] req@ffff9e35da6f6580 x1874672085897472/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829758 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829733.896930:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6580 x1874672085897472/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829758 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829733.896936:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829733.896938:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829733.896940:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085897472:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829734.023957:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829734.023968:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829734.024028:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787829734.024037:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829735.047883:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829735.047900:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829735.047904:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787829735.047926:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829736.071898:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829736.071898:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829736.071915:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787829736.071942:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829737.095906:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829737.095906:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829737.095925:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829737.095940:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829738.119952:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829738.119958:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829738.119986:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829738.119990:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829739.015955:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829739.015966:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787829739.015971:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829739.015981:0:27606:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3174 00000100:00080000:0.0:1787829739.015992:0:27606:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829739.015999:0:27606:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787829739.016047:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a1c0 x1874672085897600/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829739.016098:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829739.016104:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829739.016106:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829739.016111:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3174 00000100:00080000:0.0:1787829739.016115:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829739.016128:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829739.016144:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829739.016149:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085897600:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1787829739.016164:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718b40 x1874672085897728/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829739.016174:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829739.016178:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787829739.016179:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829739.016183:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3174 00000100:00080000:0.0:1787829739.016186:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829739.016189:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829739.016193:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829739.016203:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718000 x1874672085897856/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829739.016210:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829739.016212:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829739.016214:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829739.016220:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3174 00000800:00000100:2.0:1787829739.016221:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787829739.016223:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829739.016227:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829739.016230:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:0.0:1787829739.016243:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a580 x1874672085897984/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829739.016253:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829739.016257:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829739.016259:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829739.016265:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3174 00000100:00100000:2.0:1787829739.016269:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829739/real 1787829739] req@ffff9e35da71a1c0 x1874672085897600/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829764 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829739.016270:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829739.016279:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829739.016288:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:2.0:1787829739.016293:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a1c0 x1874672085897600/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829764 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829739.016303:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da719a40 x1874672085898112/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829739.016304:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829739.016312:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:0.0:1787829739.016318:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787829739.016319:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085897600:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1787829739.016324:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829739.016327:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829739.016334:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3174 00000100:00080000:0.0:1787829739.016337:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829739.016340:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829739.016344:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829739.016351:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7183c0 x1874672085898240/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829739.016354:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085897728:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787829739.016368:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829739.016371:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787829739.016372:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829739.016377:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3174 00000100:00080000:0.0:1787829739.016381:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1787829739.016383:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787829739.016389:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829739.016395:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829739.016397:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829739/real 1787829739] req@ffff9e35da718b40 x1874672085897728/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829764 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829739.016407:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718b40 x1874672085897728/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829764 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829739.016413:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718f00 x1874672085898368/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829739.016416:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829739.016424:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829739.016431:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085897728:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787829739.016438:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787829739.016440:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085897856:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787829739.016446:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829739.016450:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829739.016457:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3174 00000800:00000100:2.0:1787829739.016461:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787829739.016464:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829739.016472:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787829739.016473:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829739/real 1787829739] req@ffff9e35da718000 x1874672085897856/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829764 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1787829739.016482:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:2.0:1787829739.016486:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718000 x1874672085897856/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829764 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829739.016494:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1787829739.016496:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a940 x1874672085898496/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829739.016500:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829739.016507:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085897856:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787829739.016514:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787829739.016518:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085897984:192.168.203.141@tcp:8:kworker.0 00000100:00080000:0.0:1787829739.016519:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829739.016524:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829739.016529:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3174 00000100:00080000:0.0:1787829739.016532:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829739.016535:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:2.0:1787829739.016540:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:0.0:1787829739.016541:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:2.0:1787829739.016550:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829739/real 1787829739] req@ffff9e35da71a580 x1874672085897984/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829764 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829739.016554:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718780 x1874672085898624/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829739.016573:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672085897984/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829764 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829739.016580:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829739.016583:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829739.016586:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085897984:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829739.016591:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085898112:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829739.016604:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829739.016610:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829739/real 1787829739] req@ffff9e35da719a40 x1874672085898112/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829764 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829739.016618:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672085898112/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829764 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829739.016623:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829739.016625:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829739.016627:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085898112:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829739.016640:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085898240:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787829739.016650:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829739.016656:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829739/real 1787829739] req@ffff9e35da7183c0 x1874672085898240/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829764 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829739.016663:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672085898240/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829764 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829739.016673:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829739.016675:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829739.016677:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085898240:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829739.016680:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085898368:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787829739.016690:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829739.016695:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829739/real 1787829739] req@ffff9e35da718f00 x1874672085898368/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829764 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829739.016703:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672085898368/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829764 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829739.016708:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829739.016711:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829739.016713:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085898368:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829739.016716:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085898496:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829739.016725:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829739.016730:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829739/real 1787829739] req@ffff9e35da71a940 x1874672085898496/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829764 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829739.016738:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672085898496/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829764 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829739.016743:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829739.016745:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829739.016747:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085898496:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829739.016750:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085898624:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829739.016759:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829739.016764:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829739/real 1787829739] req@ffff9e35da718780 x1874672085898624/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829764 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829739.016770:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718780 x1874672085898624/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829764 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829739.016776:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829739.016778:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829739.016780:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085898624:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829739.143913:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829739.143913:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787829739.143974:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829739.143976:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829740.167936:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829740.167936:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829740.168009:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829740.168013:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829741.191909:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829741.191961:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829741.192026:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829741.192038:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829742.215950:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829742.215960:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829742.215992:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829742.216000:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829743.239912:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829743.239913:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829743.239923:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829743.239927:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829744.135870:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829744.135878:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787829744.135882:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829744.135890:0:27606:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3179 00000100:00080000:0.0:1787829744.135899:0:27606:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829744.135905:0:27606:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787829744.135938:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7192c0 x1874672085898752/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829744.135960:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829744.135963:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829744.135964:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829744.135967:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3179 00000100:00080000:0.0:1787829744.135970:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829744.135972:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829744.135980:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829744.135989:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718780 x1874672085898880/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829744.135996:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829744.135998:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787829744.135999:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829744.136002:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3179 00000100:00080000:0.0:1787829744.136003:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829744.136006:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829744.136009:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829744.136015:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a940 x1874672085899008/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829744.136023:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829744.136025:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829744.136026:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829744.136032:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3179 00000100:00100000:2.0:1787829744.136032:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085898752:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1787829744.136037:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829744.136042:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829744.136058:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:0.0:1787829744.136070:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718f00 x1874672085899136/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829744.136078:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829744.136080:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829744.136081:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829744.136084:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3179 00000100:00080000:0.0:1787829744.136088:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829744.136090:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829744.136094:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:0.0:1787829744.136108:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7183c0 x1874672085899264/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829744.136114:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787829744.136123:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829744.136125:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829744.136127:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829744.136131:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3179 00000100:00080000:0.0:1787829744.136134:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829744.136137:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829744.136139:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829744.136148:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da719a40 x1874672085899392/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829744.136154:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787829744.136158:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829744/real 1787829744] req@ffff9e35da7192c0 x1874672085898752/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829769 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829744.136159:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787829744.136162:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829744.136170:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3179 00000100:00080000:0.0:1787829744.136174:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829744.136178:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829744.136181:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829744.136186:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7192c0 x1874672085898752/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829769 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829744.136193:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a580 x1874672085899520/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829744.136198:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829744.136202:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829744.136205:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:0.0:1787829744.136208:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829744.136211:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787829744.136212:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085898752:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1787829744.136218:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3179 00000100:00080000:0.0:1787829744.136222:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787829744.136225:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085898880:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787829744.136229:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829744.136234:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:0.0:1787829744.136244:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718000 x1874672085899648/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829744.136251:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787829744.136254:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829744.136258:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829744.136259:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787829744.136263:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829744/real 1787829744] req@ffff9e35da718780 x1874672085898880/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829769 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829744.136264:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3179 00000100:00080000:0.0:1787829744.136270:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829744.136276:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829744.136283:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:2.0:1787829744.136287:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718780 x1874672085898880/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829769 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829744.136294:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718b40 x1874672085899776/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829744.136298:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829744.136307:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829744.136310:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085898880:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829744.136333:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085899008:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787829744.136346:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829744.136352:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829744/real 1787829744] req@ffff9e35da71a940 x1874672085899008/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829769 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829744.136360:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672085899008/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829769 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829744.136365:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829744.136367:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829744.136369:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085899008:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829744.136373:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085899136:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829744.136383:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829744.136389:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829744/real 1787829744] req@ffff9e35da718f00 x1874672085899136/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829769 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829744.136396:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672085899136/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829769 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829744.136401:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829744.136403:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829744.136405:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085899136:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829744.136408:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085899264:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829744.136417:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829744.136423:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829744/real 1787829744] req@ffff9e35da7183c0 x1874672085899264/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829769 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829744.136430:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672085899264/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829769 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829744.136435:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829744.136437:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829744.136439:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085899264:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829744.136443:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085899392:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787829744.136452:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829744.136458:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829744/real 1787829744] req@ffff9e35da719a40 x1874672085899392/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829769 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829744.136469:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672085899392/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829769 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829744.136474:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829744.136476:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829744.136478:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085899392:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829744.136481:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085899520:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787829744.136490:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829744.136496:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829744/real 1787829744] req@ffff9e35da71a580 x1874672085899520/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829769 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829744.136502:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672085899520/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829769 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829744.136507:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829744.136509:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829744.136511:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085899520:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829744.136522:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085899648:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829744.136537:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829744.136543:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829744/real 1787829744] req@ffff9e35da718000 x1874672085899648/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829769 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829744.136550:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718000 x1874672085899648/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829769 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829744.136560:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829744.136562:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829744.136564:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085899648:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829744.136567:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085899776:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829744.136577:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829744.136583:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829744/real 1787829744] req@ffff9e35da718b40 x1874672085899776/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829769 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829744.136590:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718b40 x1874672085899776/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829769 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829744.136595:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829744.136596:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829744.136598:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085899776:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829744.263881:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829744.263886:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829744.263895:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787829744.264036:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829745.287874:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829745.287880:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829745.287882:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829745.287938:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829746.311931:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829746.311932:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829746.311934:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829746.311993:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829747.335857:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829747.335872:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829747.335872:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787829747.335929:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829748.359899:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829748.359899:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829748.359907:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829748.359946:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829749.255976:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829749.255986:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829749.255991:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829749.256002:0:27606:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3184 00000100:00080000:1.0:1787829749.256050:0:27606:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829749.256059:0:27606:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787829749.256092:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4780 x1874672085899904/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829749.256137:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829749.256141:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829749.256143:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829749.256147:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3184 00000100:00080000:1.0:1787829749.256152:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829749.256157:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829749.256170:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829749.256188:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085899904:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1787829749.256190:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6d00 x1874672085900032/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829749.256217:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829749.256221:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829749.256224:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829749.256229:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3184 00000100:00080000:1.0:1787829749.256235:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829749.256238:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829749.256244:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829749.256258:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6940 x1874672085900160/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829749.256268:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1787829749.256276:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787829749.256277:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829749.256285:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829749.256291:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3184 00000100:00080000:1.0:1787829749.256297:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829749.256301:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829749.256306:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:1.0:1787829749.256318:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4000 x1874672085900288/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829749.256327:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787829749.256332:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829749/real 1787829749] req@ffff9e35da6f4780 x1874672085899904/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829774 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829749.256333:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829749.256338:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829749.256350:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3184 00000100:00080000:1.0:1787829749.256357:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829749.256360:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829749.256364:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:1.0:1787829749.256375:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7480 x1874672085900416/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829749.256381:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4780 x1874672085899904/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829774 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829749.256419:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829749.256423:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829749.256425:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829749.256430:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:1.0:1787829749.256430:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787829749.256435:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085899904:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787829749.256441:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3184 00000100:00080000:1.0:1787829749.256446:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829749.256450:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829749.256456:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829749.256466:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085900032:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829749.256472:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4780 x1874672085900544/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829749.256484:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829749.256488:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000800:00000100:0.0:1787829749.256491:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787829749.256492:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829749.256500:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3184 00000100:00080000:1.0:1787829749.256504:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787829749.256507:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829749/real 1787829749] req@ffff9e35da6f6d00 x1874672085900032/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829774 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829749.256511:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829749.256519:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829749.256532:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6d00 x1874672085900032/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829774 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829749.256540:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1787829749.256542:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6580 x1874672085900672/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829749.256545:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829749.256558:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085900032:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787829749.256562:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829749.256566:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:0.0:1787829749.256567:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085900160:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787829749.256569:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829749.256577:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3184 00000100:00080000:1.0:1787829749.256580:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829749.256583:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829749.256587:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000800:00000100:0.0:1787829749.256593:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829749.256597:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4f00 x1874672085900800/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829749.256605:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829749.256607:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829749.256608:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829749.256611:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3184 00000100:00100000:0.0:1787829749.256616:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829749/real 1787829749] req@ffff9e35da6f6940 x1874672085900160/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829774 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829749.256617:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829749.256623:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829749.256628:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:1.0:1787829749.256641:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f61c0 x1874672085900928/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829749.256647:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6940 x1874672085900160/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829774 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829749.256661:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829749.256666:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829749.256673:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085900160:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829749.256682:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085900288:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829749.256708:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829749.256717:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829749/real 1787829749] req@ffff9e35da6f4000 x1874672085900288/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829774 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829749.256725:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4000 x1874672085900288/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829774 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829749.256735:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829749.256737:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829749.256747:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085900288:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829749.256754:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085900416:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829749.256766:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829749.256773:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829749/real 1787829749] req@ffff9e35da6f7480 x1874672085900416/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829774 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829749.256780:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7480 x1874672085900416/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829774 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829749.256785:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829749.256787:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829749.256789:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085900416:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829749.256834:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085900544:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829749.256850:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829749.256863:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829749/real 1787829749] req@ffff9e35da6f4780 x1874672085900544/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829774 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829749.256872:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4780 x1874672085900544/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829774 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829749.256878:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829749.256880:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829749.256882:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085900544:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829749.256886:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085900672:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829749.256897:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829749.256902:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829749/real 1787829749] req@ffff9e35da6f6580 x1874672085900672/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829774 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829749.256909:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6580 x1874672085900672/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829774 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829749.256914:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829749.256916:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829749.256918:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085900672:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829749.256921:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085900800:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829749.256930:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829749.256936:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829749/real 1787829749] req@ffff9e35da6f4f00 x1874672085900800/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829774 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829749.256943:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4f00 x1874672085900800/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829774 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829749.256948:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829749.256950:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829749.256952:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085900800:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829749.256955:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085900928:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829749.256964:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829749.256970:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829749/real 1787829749] req@ffff9e35da6f61c0 x1874672085900928/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829774 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829749.256977:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f61c0 x1874672085900928/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829774 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829749.256982:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829749.256984:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829749.256985:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085900928:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829749.383936:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829749.383936:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829749.383955:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829749.383984:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829750.407944:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829750.407977:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829750.408021:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:0.0:1787829750.408073:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829751.432008:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829751.432008:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829751.432015:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829751.432111:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829752.455973:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829752.455983:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829752.456041:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829752.456108:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829753.479935:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829753.479935:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829753.479935:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787829753.480062:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829754.375988:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829754.376000:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829754.376004:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829754.376016:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3189 00000100:00080000:1.0:1787829754.376027:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829754.376034:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787829754.376077:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7840 x1874672085901056/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829754.376106:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829754.376109:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829754.376110:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829754.376113:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3189 00000100:00080000:1.0:1787829754.376116:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829754.376118:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829754.376128:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829754.376137:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5a40 x1874672085901184/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829754.376145:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829754.376148:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829754.376150:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829754.376152:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3189 00000100:00080000:1.0:1787829754.376155:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829754.376158:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829754.376160:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829754.376169:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5e00 x1874672085901312/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829754.376181:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829754.376187:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:0.0:1787829754.376188:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085901056:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787829754.376190:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829754.376203:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3189 00000100:00080000:1.0:1787829754.376206:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829754.376210:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829754.376214:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:1.0:1787829754.376231:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4b40 x1874672085901440/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829754.376238:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829754.376242:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829754.376245:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829754.376250:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3189 00000100:00080000:1.0:1787829754.376265:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1787829754.376270:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787829754.376276:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829754.376286:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:1.0:1787829754.376307:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f70c0 x1874672085901568/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829754.376317:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829754/real 1787829754] req@ffff9e35da6f7840 x1874672085901056/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829779 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829754.376320:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829754.376332:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829754.376338:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829754.376345:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3189 00000100:00080000:1.0:1787829754.376354:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829754.376358:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829754.376364:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829754.376371:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7840 x1874672085901056/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829779 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829754.376379:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f43c0 x1874672085901696/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829754.376426:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829754.376427:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829754.376430:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829754.376433:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829754.376438:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3189 00000100:00080000:1.0:1787829754.376444:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829754.376445:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:1.0:1787829754.376450:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829754.376455:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829754.376461:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085901056:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1787829754.376467:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f61c0 x1874672085901824/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829754.376478:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085901184:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787829754.376479:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829754.376496:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829754.376498:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829754.376503:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3189 00000100:00080000:1.0:1787829754.376506:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829754.376509:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829754.376513:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:1.0:1787829754.376523:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4f00 x1874672085901952/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829754.376534:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787829754.376536:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829754.376541:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829754.376543:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829754.376550:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3189 00000100:00080000:1.0:1787829754.376554:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829754.376558:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787829754.376562:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829754/real 1787829754] req@ffff9e35da6f5a40 x1874672085901184/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829779 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1787829754.376565:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:1.0:1787829754.376584:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6580 x1874672085902080/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829754.376589:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5a40 x1874672085901184/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829779 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829754.376648:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829754.376652:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829754.376655:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085901184:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829754.376690:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085901312:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829754.376708:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829754.376715:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829754/real 1787829754] req@ffff9e35da6f5e00 x1874672085901312/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829779 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829754.376723:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5e00 x1874672085901312/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829779 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829754.376728:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829754.376731:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829754.376733:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085901312:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829754.376737:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085901440:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829754.376747:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829754.376753:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829754/real 1787829754] req@ffff9e35da6f4b40 x1874672085901440/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829779 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829754.376764:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4b40 x1874672085901440/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829779 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829754.376769:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829754.376771:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829754.376773:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085901440:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829754.376777:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085901568:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829754.376788:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829754.376853:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829754/real 1787829754] req@ffff9e35da6f70c0 x1874672085901568/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829779 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829754.376864:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f70c0 x1874672085901568/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829779 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829754.376871:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829754.376873:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829754.376875:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085901568:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829754.376880:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085901696:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829754.376896:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829754.376904:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829754/real 1787829754] req@ffff9e35da6f43c0 x1874672085901696/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829779 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829754.376911:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f43c0 x1874672085901696/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829779 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829754.376921:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829754.376923:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829754.376925:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085901696:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829754.376929:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085901824:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829754.376939:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829754.376945:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829754/real 1787829754] req@ffff9e35da6f61c0 x1874672085901824/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829779 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829754.376952:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f61c0 x1874672085901824/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829779 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829754.376957:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829754.376959:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829754.376961:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085901824:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829754.376964:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085901952:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829754.376990:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829754.376999:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829754/real 1787829754] req@ffff9e35da6f4f00 x1874672085901952/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829779 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829754.377007:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4f00 x1874672085901952/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829779 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829754.377013:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829754.377015:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829754.377017:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085901952:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829754.377022:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085902080:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829754.377034:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829754.377040:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829754/real 1787829754] req@ffff9e35da6f6580 x1874672085902080/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829779 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829754.377047:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6580 x1874672085902080/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829779 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829754.377052:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829754.377054:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829754.377056:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085902080:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829754.503967:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829754.503968:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829754.503984:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829754.504070:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829755.527880:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829755.527940:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829755.527960:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829755.527963:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829756.551872:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829756.551874:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829756.551909:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:0.0:1787829756.551989:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829757.575852:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829757.575876:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829757.575902:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829757.575931:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829758.599899:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829758.599942:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829758.599977:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829758.600026:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829759.495992:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829759.496007:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787829759.496013:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829759.496026:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3194 00000100:00080000:0.0:1787829759.496041:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829759.496050:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787829759.496111:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a1c0 x1874672085902208/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829759.496150:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829759.496154:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829759.496157:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829759.496162:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3194 00000100:00080000:0.0:1787829759.496167:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829759.496171:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829759.496184:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829759.496203:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718b40 x1874672085902336/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829759.496212:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085902208:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1787829759.496220:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829759.496226:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787829759.496228:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829759.496234:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3194 00000100:00080000:0.0:1787829759.496239:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829759.496244:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829759.496250:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829759.496266:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718000 x1874672085902464/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829759.496277:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829759.496282:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829759.496284:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829759.496289:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3194 00000100:00080000:0.0:1787829759.496293:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1787829759.496300:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787829759.496301:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829759.496321:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:0.0:1787829759.496334:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a580 x1874672085902592/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829759.496342:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829759.496345:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829759.496346:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829759.496350:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3194 00000100:00080000:0.0:1787829759.496353:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829759.496356:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829759.496360:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:0.0:1787829759.496376:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da719a40 x1874672085902720/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829759.496376:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829759/real 1787829759] req@ffff9e35da71a1c0 x1874672085902208/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829784 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829759.496385:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829759.496389:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829759.496392:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829759.496401:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3194 00000100:00080000:0.0:1787829759.496406:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829759.496414:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829759.496420:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829759.496428:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a1c0 x1874672085902208/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829784 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829759.496436:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7183c0 x1874672085902848/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829759.496447:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829759.496452:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829759.496456:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:0.0:1787829759.496459:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787829759.496463:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787829759.496464:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085902208:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1787829759.496474:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3194 00000100:00080000:0.0:1787829759.496480:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829759.496484:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829759.496490:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829759.496501:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718f00 x1874672085902976/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829759.496505:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085902336:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787829759.496514:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829759.496527:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829759.496530:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829759.496537:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3194 00000800:00000100:2.0:1787829759.496542:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787829759.496545:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829759.496553:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829759.496559:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:2.0:1787829759.496559:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829759/real 1787829759] req@ffff9e35da718b40 x1874672085902336/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829784 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829759.496579:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a940 x1874672085903104/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829759.496581:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718b40 x1874672085902336/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829784 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829759.496602:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829759.496606:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829759.496607:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829759.496609:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829759.496617:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3194 00000100:00080000:2.0:1787829759.496621:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:0.0:1787829759.496623:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787829759.496627:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085902336:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787829759.496630:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829759.496639:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:2.0:1787829759.496641:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085902464:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829759.496652:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718780 x1874672085903232/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829759.496665:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829759.496676:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829759/real 1787829759] req@ffff9e35da718000 x1874672085902464/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829784 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829759.496687:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718000 x1874672085902464/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829784 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829759.496693:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829759.496696:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829759.496698:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085902464:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829759.496702:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085902592:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829759.496713:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829759.496718:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829759/real 1787829759] req@ffff9e35da71a580 x1874672085902592/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829784 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829759.496726:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672085902592/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829784 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829759.496731:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829759.496733:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829759.496735:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085902592:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829759.496740:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085902720:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829759.496757:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829759.496768:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829759/real 1787829759] req@ffff9e35da719a40 x1874672085902720/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829784 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829759.496782:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672085902720/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829784 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829759.496792:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829759.496795:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829759.496835:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085902720:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829759.496839:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085902848:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787829759.496856:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829759.496864:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829759/real 1787829759] req@ffff9e35da7183c0 x1874672085902848/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829784 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829759.496873:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672085902848/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829784 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829759.496879:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829759.496881:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829759.496883:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085902848:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829759.496899:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085902976:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787829759.496910:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829759.496916:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829759/real 1787829759] req@ffff9e35da718f00 x1874672085902976/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829784 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829759.496923:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672085902976/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829784 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829759.496934:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829759.496936:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829759.496955:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085902976:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829759.496960:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085903104:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829759.496974:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829759.496982:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829759/real 1787829759] req@ffff9e35da71a940 x1874672085903104/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829784 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829759.496989:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672085903104/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829784 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829759.496995:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829759.496998:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829759.497000:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085903104:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829759.497004:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085903232:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829759.497013:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829759.497019:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829759/real 1787829759] req@ffff9e35da718780 x1874672085903232/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829784 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829759.497026:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718780 x1874672085903232/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829784 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829759.497031:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829759.497033:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829759.497035:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085903232:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829759.623919:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829759.623919:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829759.623971:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787829759.623978:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829760.647889:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829760.647890:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829760.647925:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829760.648068:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829761.671896:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829761.671896:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829761.671907:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829761.671966:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829762.695899:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829762.695899:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829762.695926:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829762.695990:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829763.719900:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829763.719900:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829763.719919:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829763.719962:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829764.615942:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829764.615950:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829764.615954:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829764.615963:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3199 00000100:00080000:2.0:1787829764.615973:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829764.615978:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787829764.616018:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e940 x1874672085903360/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829764.616040:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829764.616043:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829764.616044:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829764.616047:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3199 00000100:00080000:2.0:1787829764.616050:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829764.616053:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829764.616062:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829764.616071:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f0c0 x1874672085903488/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829764.616077:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085903360:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1787829764.616083:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829764.616098:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829764.616099:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829764.616103:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3199 00000100:00080000:2.0:1787829764.616105:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829764.616107:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829764.616111:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829764.616125:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c000 x1874672085903616/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829764.616132:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829764.616135:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829764.616136:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829764.616138:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3199 00000100:00080000:2.0:1787829764.616141:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829764.616144:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829764.616147:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000800:00000100:0.0:1787829764.616152:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829764.616156:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cb40 x1874672085903744/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829764.616164:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829764.616168:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829764.616170:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829764.616175:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3199 00000100:00080000:2.0:1787829764.616179:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829764.616184:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829764.616190:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:0.0:1787829764.616191:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829764/real 1787829764] req@ffff9e35c498e940 x1874672085903360/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829789 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829764.616208:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498fc00 x1874672085903872/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829764.616210:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e940 x1874672085903360/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829789 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829764.616241:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829764.616241:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829764.616248:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829764.616249:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:2.0:1787829764.616250:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787829764.616254:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085903360:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1787829764.616265:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3199 00000100:00080000:2.0:1787829764.616270:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829764.616273:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787829764.616278:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085903488:192.168.203.141@tcp:38:kworker.0 00000008:00100000:2.0:1787829764.616278:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829764.616293:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498d680 x1874672085904000/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829764.616302:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1787829764.616303:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829764.616307:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829764.616309:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829764.616313:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3199 00000100:00100000:0.0:1787829764.616316:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829764/real 1787829764] req@ffff9e35c498f0c0 x1874672085903488/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829789 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829764.616318:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829764.616326:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829764.616333:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829764.616337:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f0c0 x1874672085903488/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829789 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829764.616345:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e940 x1874672085904128/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829764.616347:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829764.616355:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829764.616362:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085903488:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1787829764.616368:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787829764.616369:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085903616:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1787829764.616377:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829764.616380:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829764.616388:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3199 00000800:00000100:0.0:1787829764.616389:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1787829764.616397:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787829764.616399:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829764/real 1787829764] req@ffff9e35c498c000 x1874672085903616/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829789 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829764.616406:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829764.616415:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:0.0:1787829764.616416:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c000 x1874672085903616/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829789 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829764.616424:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829764.616429:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829764.616430:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498de00 x1874672085904256/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829764.616433:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085903616:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829764.616445:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085903744:192.168.203.141@tcp:8:kworker.0 00000100:00080000:2.0:1787829764.616448:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829764.616452:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829764.616454:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829764.616459:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3199 00000100:00080000:2.0:1787829764.616462:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1787829764.616464:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1787829764.616468:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829764.616474:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:0.0:1787829764.616476:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829764/real 1787829764] req@ffff9e35c498cb40 x1874672085903744/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829789 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829764.616485:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f840 x1874672085904384/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829764.616487:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498cb40 x1874672085903744/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829789 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829764.616510:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829764.616513:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829764.616515:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085903744:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829764.616520:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085903872:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829764.616532:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829764.616539:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829764/real 1787829764] req@ffff9e35c498fc00 x1874672085903872/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829789 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829764.616546:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498fc00 x1874672085903872/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829789 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829764.616551:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829764.616553:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829764.616555:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085903872:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829764.616569:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085904000:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829764.616579:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829764.616585:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829764/real 1787829764] req@ffff9e35c498d680 x1874672085904000/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829789 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829764.616592:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498d680 x1874672085904000/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829789 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829764.616597:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829764.616599:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829764.616601:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085904000:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829764.616604:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085904128:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829764.616614:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829764.616620:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829764/real 1787829764] req@ffff9e35c498e940 x1874672085904128/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829789 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829764.616626:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e940 x1874672085904128/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829789 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829764.616632:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829764.616634:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829764.616636:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085904128:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829764.616639:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085904256:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829764.616648:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829764.616653:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829764/real 1787829764] req@ffff9e35c498de00 x1874672085904256/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829789 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829764.616660:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498de00 x1874672085904256/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829789 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829764.616665:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829764.616667:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829764.616669:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085904256:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829764.616672:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085904384:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829764.616681:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829764.616686:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829764/real 1787829764] req@ffff9e35c498f840 x1874672085904384/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829789 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829764.616693:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f840 x1874672085904384/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829789 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829764.616698:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829764.616700:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829764.616702:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085904384:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829764.743870:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829764.743870:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829764.743897:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829764.743918:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829765.767924:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829765.767924:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829765.767924:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787829765.768095:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829766.791902:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829766.791931:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829766.791956:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829766.792041:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829767.815913:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829767.815913:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829767.815970:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829767.815987:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829768.839907:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829768.839932:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787829768.839991:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829768.839992:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00080000:1.0:1787829769.735991:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829769.736001:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829769.736006:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829769.736016:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3205 00000100:00080000:1.0:1787829769.736028:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829769.736037:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787829769.736077:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4780 x1874672085904512/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829769.736110:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829769.736114:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829769.736117:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829769.736122:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3205 00000100:00080000:1.0:1787829769.736126:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829769.736131:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829769.736143:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829769.736153:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085904512:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1787829769.736162:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7480 x1874672085904640/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829769.736175:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829769.736179:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829769.736182:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829769.736186:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3205 00000100:00080000:1.0:1787829769.736209:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829769.736213:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829769.736219:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1787829769.736221:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829769.736237:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4000 x1874672085904768/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829769.736255:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829769.736259:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829769.736262:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787829769.736267:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829769/real 1787829769] req@ffff9e35da6f4780 x1874672085904512/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829794 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829769.736279:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3205 00000100:00100000:0.0:1787829769.736284:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4780 x1874672085904512/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829794 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829769.736286:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829769.736296:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:0.0:1787829769.736300:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:1.0:1787829769.736304:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00080000:0.0:1787829769.736309:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829769.736316:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085904512:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1787829769.736322:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6940 x1874672085904896/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829769.736335:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829769.736339:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829769.736341:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787829769.736345:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085904640:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787829769.736349:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3205 00000100:00080000:1.0:1787829769.736353:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829769.736363:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829769.736369:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000800:00000100:0.0:1787829769.736383:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829769.736387:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6d00 x1874672085905024/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829769.736402:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787829769.736403:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829769/real 1787829769] req@ffff9e35da6f7480 x1874672085904640/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829794 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829769.736411:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829769.736417:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787829769.736422:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7480 x1874672085904640/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829794 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829769.736428:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3205 00000100:00080000:0.0:1787829769.736433:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829769.736436:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829769.736439:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:1.0:1787829769.736444:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787829769.736445:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085904640:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829769.736455:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085904768:192.168.203.141@tcp:38:kworker.0 00000008:00100000:1.0:1787829769.736456:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1787829769.736473:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829769.736476:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4780 x1874672085905152/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829769.736485:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829769/real 1787829769] req@ffff9e35da6f4000 x1874672085904768/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829794 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829769.736498:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4000 x1874672085904768/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829794 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829769.736506:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829769.736506:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829769.736515:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:1.0:1787829769.736516:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829769.736521:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787829769.736523:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085904768:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787829769.736529:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3205 00000100:00080000:1.0:1787829769.736536:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787829769.736537:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085904896:192.168.203.141@tcp:8:kworker.0 00000100:00080000:1.0:1787829769.736543:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829769.736551:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1787829769.736569:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829769.736578:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6580 x1874672085905280/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829769.736586:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829769/real 1787829769] req@ffff9e35da6f6940 x1874672085904896/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829794 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829769.736594:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829769.736604:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829769.736606:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787829769.736609:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6940 x1874672085904896/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829794 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829769.736615:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3205 00000100:00080000:0.0:1787829769.736621:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829769.736623:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829769.736626:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:1.0:1787829769.736634:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787829769.736635:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085904896:192.168.203.141@tcp:8:kworker.0 00000008:00100000:1.0:1787829769.736647:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:1.0:1787829769.736662:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4f00 x1874672085905408/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829769.736667:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085905024:192.168.203.141@tcp:8:kworker.0 00000100:00080000:1.0:1787829769.736688:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1787829769.736694:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787829769.736695:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829769.736700:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787829769.736708:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829769/real 1787829769] req@ffff9e35da6f6d00 x1874672085905024/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829794 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829769.736708:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3205 00000100:00080000:1.0:1787829769.736716:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787829769.736721:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6d00 x1874672085905024/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829794 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829769.736725:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829769.736736:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00080000:0.0:1787829769.736737:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829769.736744:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829769.736747:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085905024:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829769.736753:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085905152:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829769.736757:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f61c0 x1874672085905536/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829769.736769:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829769.736780:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829769/real 1787829769] req@ffff9e35da6f4780 x1874672085905152/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829794 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829769.736788:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4780 x1874672085905152/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829794 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829769.736794:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829769.736843:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829769.736846:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085905152:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829769.736850:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085905280:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829769.736863:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829769.736891:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829769/real 1787829769] req@ffff9e35da6f6580 x1874672085905280/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829794 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829769.736904:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6580 x1874672085905280/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829794 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829769.736910:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829769.736912:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829769.736914:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085905280:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829769.736925:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085905408:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829769.736938:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829769.736945:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829769/real 1787829769] req@ffff9e35da6f4f00 x1874672085905408/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829794 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829769.736953:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4f00 x1874672085905408/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829794 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829769.736958:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829769.736960:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829769.736961:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085905408:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829769.736965:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085905536:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829769.736975:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829769.736980:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829769/real 1787829769] req@ffff9e35da6f61c0 x1874672085905536/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829794 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829769.736987:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f61c0 x1874672085905536/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829794 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829769.736996:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829769.736998:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829769.737000:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085905536:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829769.863960:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829769.863972:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:3.0:1787829769.864092:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829769.864146:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829770.887961:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829770.888002:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829770.888031:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829770.888041:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829771.911930:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829771.911930:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829771.911940:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787829771.911983:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829772.935918:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829772.935934:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829772.935940:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829772.935954:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829773.959902:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829773.959905:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829773.959915:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787829773.959956:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829774.855910:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829774.855920:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829774.855925:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829774.855935:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3210 00000100:00080000:2.0:1787829774.855946:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829774.855953:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787829774.855989:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c3c0 x1874672085905664/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829774.856018:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829774.856027:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829774.856029:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829774.856034:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3210 00000100:00080000:2.0:1787829774.856038:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829774.856043:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829774.856054:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829774.856070:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c780 x1874672085905792/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829774.856075:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085905664:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1787829774.856088:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829774.856092:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829774.856094:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829774.856099:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3210 00000100:00080000:2.0:1787829774.856104:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829774.856109:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829774.856115:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829774.856128:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e1c0 x1874672085905920/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829774.856139:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1787829774.856145:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1787829774.856146:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829774.856150:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829774.856155:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3210 00000100:00080000:2.0:1787829774.856161:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829774.856165:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829774.856171:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:2.0:1787829774.856185:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f480 x1874672085906048/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829774.856191:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829774/real 1787829774] req@ffff9e35c498c3c0 x1874672085905664/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829799 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829774.856204:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829774.856210:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829774.856213:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787829774.856218:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c3c0 x1874672085905664/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829799 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829774.856219:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3210 00000100:00080000:2.0:1787829774.856226:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829774.856234:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:0.0:1787829774.856237:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:2.0:1787829774.856241:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00080000:0.0:1787829774.856245:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829774.856251:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085905664:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1787829774.856260:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e580 x1874672085906176/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829774.856272:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787829774.856278:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085905792:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1787829774.856278:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829774.856283:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829774.856290:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3210 00000100:00080000:2.0:1787829774.856293:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829774.856298:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829774.856303:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1787829774.856309:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829774.856328:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cf00 x1874672085906304/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829774.856329:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829774/real 1787829774] req@ffff9e35c498c780 x1874672085905792/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829799 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829774.856363:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829774.856371:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829774.856374:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787829774.856377:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c780 x1874672085905792/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829799 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829774.856382:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3210 00000100:00080000:2.0:1787829774.856390:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829774.856393:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829774.856397:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:0.0:1787829774.856402:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829774.856407:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085905792:192.168.203.141@tcp:38:kworker.0 00000008:00100000:2.0:1787829774.856407:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829774.856419:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085905920:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829774.856426:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498ed00 x1874672085906432/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829774.856437:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1787829774.856442:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1787829774.856444:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829774.856448:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829774.856454:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3210 00000100:00100000:0.0:1787829774.856458:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829774/real 1787829774] req@ffff9e35c498e1c0 x1874672085905920/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829799 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829774.856461:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829774.856472:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787829774.856478:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e1c0 x1874672085905920/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829799 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1787829774.856480:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:2.0:1787829774.856495:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c3c0 x1874672085906560/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829774.856505:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829774.856510:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829774.856517:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085905920:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1787829774.856527:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787829774.856530:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085906048:192.168.203.141@tcp:8:kworker.0 00000100:00080000:2.0:1787829774.856534:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829774.856538:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829774.856544:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3210 00000100:00080000:2.0:1787829774.856549:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1787829774.856556:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1787829774.856557:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829774.856566:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:0.0:1787829774.856573:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829774/real 1787829774] req@ffff9e35c498f480 x1874672085906048/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829799 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829774.856582:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f840 x1874672085906688/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829774.856591:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f480 x1874672085906048/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829799 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829774.856608:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829774.856611:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829774.856615:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085906048:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829774.856621:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085906176:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829774.856637:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829774.856646:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829774/real 1787829774] req@ffff9e35c498e580 x1874672085906176/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829799 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829774.856657:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e580 x1874672085906176/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829799 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829774.856665:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829774.856668:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829774.856671:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085906176:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829774.856689:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085906304:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829774.856704:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829774.856715:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829774/real 1787829774] req@ffff9e35c498cf00 x1874672085906304/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829799 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829774.856727:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498cf00 x1874672085906304/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829799 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829774.856736:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829774.856739:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829774.856743:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085906304:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829774.856748:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085906432:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829774.856766:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829774.856776:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829774/real 1787829774] req@ffff9e35c498ed00 x1874672085906432/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829799 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829774.856787:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498ed00 x1874672085906432/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829799 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829774.856795:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829774.856823:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829774.856845:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085906432:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829774.856852:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085906560:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829774.856871:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829774.856881:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829774/real 1787829774] req@ffff9e35c498c3c0 x1874672085906560/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829799 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829774.856892:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c3c0 x1874672085906560/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829799 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829774.856900:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829774.856903:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829774.856906:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085906560:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829774.856911:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085906688:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829774.856925:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829774.856935:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829774/real 1787829774] req@ffff9e35c498f840 x1874672085906688/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829799 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829774.856944:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f840 x1874672085906688/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829799 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829774.856952:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829774.856956:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829774.856959:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085906688:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829774.983913:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829774.983913:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829774.983940:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829774.983962:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829776.007883:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829776.007883:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829776.007884:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787829776.007923:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829777.031943:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829777.031958:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829777.031960:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829777.031999:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829778.055898:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829778.055899:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829778.055899:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829778.055935:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829779.079911:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829779.079911:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829779.079923:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787829779.079956:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829779.975920:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829779.975929:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829779.975933:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829779.975943:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3215 00000100:00080000:2.0:1787829779.975953:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829779.975958:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787829779.975990:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498de00 x1874672085906816/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829779.976014:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829779.976017:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829779.976018:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829779.976021:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3215 00000100:00080000:2.0:1787829779.976023:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829779.976025:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829779.976033:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829779.976040:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e940 x1874672085906944/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829779.976054:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829779.976056:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829779.976057:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829779.976060:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3215 00000100:00080000:2.0:1787829779.976062:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829779.976064:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829779.976068:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829779.976077:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085906816:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1787829779.976097:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498d680 x1874672085907072/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829779.976108:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829779.976111:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829779.976113:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829779.976116:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3215 00000100:00080000:2.0:1787829779.976119:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829779.976122:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829779.976125:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:2.0:1787829779.976135:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498fc00 x1874672085907200/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829779.976141:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829779.976143:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829779.976144:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829779.976147:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3215 00000100:00080000:2.0:1787829779.976150:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829779.976154:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:0.0:1787829779.976159:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:2.0:1787829779.976160:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:2.0:1787829779.976186:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cb40 x1874672085907328/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829779.976195:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829779.976198:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829779.976199:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787829779.976204:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829779/real 1787829779] req@ffff9e35c498de00 x1874672085906816/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829804 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829779.976218:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3215 00000100:00080000:2.0:1787829779.976222:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787829779.976224:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498de00 x1874672085906816/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829804 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829779.976228:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829779.976232:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829779.976247:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c000 x1874672085907456/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829779.976248:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829779.976257:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829779.976265:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085906816:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1787829779.976268:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787829779.976275:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085906944:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1787829779.976284:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829779.976286:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829779.976289:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3215 00000100:00080000:2.0:1787829779.976292:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829779.976295:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829779.976298:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1787829779.976303:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829779.976309:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f0c0 x1874672085907584/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829779.976316:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829779.976318:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829779.976319:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829779.976322:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3215 00000100:00080000:2.0:1787829779.976324:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829779.976326:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829779.976329:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:0.0:1787829779.976336:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829779/real 1787829779] req@ffff9e35c498e940 x1874672085906944/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829804 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829779.976348:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f840 x1874672085907712/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829779.976358:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829779.976361:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829779.976363:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829779.976368:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3215 00000100:00100000:0.0:1787829779.976373:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e940 x1874672085906944/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829804 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829779.976375:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829779.976382:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829779.976389:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00080000:0.0:1787829779.976392:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829779.976398:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829779.976401:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c3c0 x1874672085907840/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829779.976406:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085906944:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829779.976437:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085907072:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829779.976453:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829779.976461:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829779/real 1787829779] req@ffff9e35c498d680 x1874672085907072/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829804 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829779.976468:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498d680 x1874672085907072/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829804 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829779.976473:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829779.976475:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829779.976477:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085907072:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829779.976489:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085907200:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829779.976500:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829779.976512:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829779/real 1787829779] req@ffff9e35c498fc00 x1874672085907200/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829804 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829779.976519:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498fc00 x1874672085907200/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829804 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829779.976525:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829779.976530:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829779.976532:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085907200:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829779.976536:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085907328:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829779.976545:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829779.976551:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829779/real 1787829779] req@ffff9e35c498cb40 x1874672085907328/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829804 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829779.976557:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498cb40 x1874672085907328/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829804 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829779.976561:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829779.976563:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829779.976565:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085907328:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829779.976568:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085907456:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829779.976576:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829779.976581:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829779/real 1787829779] req@ffff9e35c498c000 x1874672085907456/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829804 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829779.976597:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c000 x1874672085907456/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829804 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829779.976605:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829779.976608:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829779.976611:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085907456:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829779.976616:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085907584:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829779.976634:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829779.976643:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829779/real 1787829779] req@ffff9e35c498f0c0 x1874672085907584/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829804 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829779.976654:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f0c0 x1874672085907584/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829804 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829779.976661:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829779.976664:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829779.976667:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085907584:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829779.976685:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085907712:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829779.976699:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829779.976706:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829779/real 1787829779] req@ffff9e35c498f840 x1874672085907712/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829804 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829779.976713:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f840 x1874672085907712/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829804 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829779.976718:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829779.976720:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829779.976722:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085907712:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829779.976725:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085907840:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829779.976733:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829779.976739:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829779/real 1787829779] req@ffff9e35c498c3c0 x1874672085907840/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829804 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829779.976745:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c3c0 x1874672085907840/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829804 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829779.976749:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829779.976751:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829779.976752:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085907840:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829780.103885:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829780.103886:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829780.103907:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787829780.103921:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829781.127906:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829781.127906:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829781.127919:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829781.127948:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829782.151878:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829782.151878:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829782.151888:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787829782.151917:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829783.175897:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829783.175897:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829783.175909:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787829783.175947:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829784.199865:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829784.199865:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829784.199905:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829784.199977:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00080000:2.0:1787829785.095939:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829785.095946:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829785.095950:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829785.095961:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3220 00000100:00080000:2.0:1787829785.095971:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829785.095978:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787829785.096011:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498ed00 x1874672085907968/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829785.096039:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829785.096043:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829785.096045:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829785.096049:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3220 00000100:00080000:2.0:1787829785.096053:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829785.096058:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829785.096068:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829785.096085:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085907968:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1787829785.096085:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cf00 x1874672085908096/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829785.096103:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829785.096106:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829785.096108:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829785.096112:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3220 00000100:00080000:2.0:1787829785.096116:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829785.096120:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829785.096125:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829785.096135:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e580 x1874672085908224/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829785.096144:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829785.096147:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829785.096149:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829785.096153:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3220 00000100:00080000:2.0:1787829785.096156:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829785.096160:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829785.096165:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:2.0:1787829785.096175:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f480 x1874672085908352/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829785.096181:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085908096:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1787829785.096188:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829785.096192:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829785.096195:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829785.096199:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3220 00000100:00080000:2.0:1787829785.096203:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829785.096207:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829785.096212:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:0.0:1787829785.096217:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085908224:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829785.096225:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e1c0 x1874672085908480/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829785.096236:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829785.096239:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:0.0:1787829785.096240:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085908352:192.168.203.141@tcp:8:kworker.0 00000100:00080000:2.0:1787829785.096242:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829785.096250:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3220 00000100:00080000:2.0:1787829785.096254:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829785.096259:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787829785.096262:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085908480:192.168.203.141@tcp:8:kworker.0 00000008:00100000:2.0:1787829785.096265:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829785.096279:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c780 x1874672085908608/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829785.096289:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829785.096292:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829785.096294:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787829785.096296:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085908608:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1787829785.096302:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3220 00000100:00080000:2.0:1787829785.096308:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829785.096312:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829785.096317:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829785.096330:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c3c0 x1874672085908736/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829785.096345:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829785.096349:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829785.096351:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787829785.096356:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085908736:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1787829785.096357:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3220 00000100:00080000:2.0:1787829785.096364:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829785.096369:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829785.096374:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:2.0:1787829785.096385:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f840 x1874672085908864/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829785.096398:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829785.096401:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829785.096404:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787829785.096408:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085908864:192.168.203.141@tcp:8:kworker.0 00000100:00080000:2.0:1787829785.096409:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3220 00000100:00080000:2.0:1787829785.096416:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829785.096420:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829785.096425:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:2.0:1787829785.096436:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f0c0 x1874672085908992/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829785.096461:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085908992:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829785.223890:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787829785.223925:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829785.223991:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:3.0:1787829785.224029:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787829786.247878:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829786.247881:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829786.247882:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787829786.247984:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829787.271878:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829787.271902:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829787.271929:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829787.271946:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1787829788.168055:0:2396:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.141:988 00000400:00000100:3.0:1787829788.168094:0:2396:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.141@tcp at host 192.168.203.141 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1787829788.168113:0:2396:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1787829788.168168:0:2396:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000800:00000100:3.0:1787829788.168183:0:2396:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1787829788.168193:0:2396:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1787829788.168202:0:2396:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1787829788.168211:0:2396:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1787829788.168222:0:2396:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1787829788.168229:0:2396:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1787829788.168237:0:2396:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:0.0:1787829788.168337:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829785/real 1787829788] req@ffff9e35c498f0c0 x1874672085908992/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829810 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829788.168363:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f0c0 x1874672085908992/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829810 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829788.168376:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829788.168382:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829788.168386:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085908992:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829788.168394:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829785/real 1787829788] req@ffff9e35c498f840 x1874672085908864/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829810 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829788.168408:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f840 x1874672085908864/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829810 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829788.168418:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829788.168422:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829788.168426:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085908864:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829788.168432:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829785/real 1787829788] req@ffff9e35c498c3c0 x1874672085908736/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829810 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829788.168445:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c3c0 x1874672085908736/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829810 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829788.168453:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829788.168457:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829788.168460:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085908736:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829788.168467:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829785/real 1787829788] req@ffff9e35c498c780 x1874672085908608/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829810 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829788.168478:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c780 x1874672085908608/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829810 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829788.168487:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829788.168491:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829788.168495:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085908608:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829788.168502:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829785/real 1787829788] req@ffff9e35c498f480 x1874672085908352/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829810 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829788.168515:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f480 x1874672085908352/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829810 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829788.168524:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829788.168528:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829788.168532:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085908352:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829788.168538:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829785/real 1787829788] req@ffff9e35c498e1c0 x1874672085908480/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829810 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829788.168552:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e1c0 x1874672085908480/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829810 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829788.168561:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829788.168565:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829788.168568:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085908480:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829788.168577:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829785/real 1787829788] req@ffff9e35c498cf00 x1874672085908096/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829810 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829788.168588:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498cf00 x1874672085908096/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829810 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829788.168602:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829788.168606:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829788.168609:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085908096:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829788.168616:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829785/real 1787829788] req@ffff9e35c498e580 x1874672085908224/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829810 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829788.168627:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e580 x1874672085908224/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829810 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829788.168635:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829788.168639:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829788.168642:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085908224:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829788.168648:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829785/real 1787829788] req@ffff9e35c498ed00 x1874672085907968/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829810 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829788.168660:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498ed00 x1874672085907968/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829810 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829788.168668:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829788.168671:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829788.168674:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085907968:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1787829788.295880:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829788.295888:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829788.295926:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787829788.295947:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829789.319910:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829789.319910:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829789.319935:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787829789.319964:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829790.215911:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829790.215922:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829790.215927:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829790.215937:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3225 00000100:00080000:1.0:1787829790.215958:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829790.215963:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787829790.215992:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f43c0 x1874672085909120/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829790.216026:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829790.216029:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829790.216031:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829790.216034:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3225 00000100:00080000:1.0:1787829790.216036:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829790.216039:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829790.216048:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829790.216059:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f70c0 x1874672085909248/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829790.216066:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829790.216069:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829790.216070:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829790.216084:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3225 00000100:00080000:1.0:1787829790.216086:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829790.216088:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829790.216092:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829790.216099:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085909120:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1787829790.216102:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4b40 x1874672085909376/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829790.216117:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829790.216120:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829790.216122:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829790.216127:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3225 00000100:00080000:1.0:1787829790.216130:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829790.216135:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829790.216141:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:1.0:1787829790.216154:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5e00 x1874672085909504/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829790.216165:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1787829790.216176:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787829790.216179:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829790.216183:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829790.216190:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3225 00000100:00080000:1.0:1787829790.216194:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829790.216198:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829790.216203:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:1.0:1787829790.216214:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5a40 x1874672085909632/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829790.216221:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829790.216224:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829790.216226:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787829790.216231:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829790/real 1787829790] req@ffff9e35da6f43c0 x1874672085909120/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829815 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829790.216233:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3225 00000100:00080000:1.0:1787829790.216241:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829790.216249:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829790.216256:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829790.216263:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f43c0 x1874672085909120/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829815 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829790.216272:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7840 x1874672085909760/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829790.216278:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829790.216288:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:1.0:1787829790.216289:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829790.216294:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:0.0:1787829790.216297:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085909120:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787829790.216298:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829790.216307:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3225 00000100:00080000:1.0:1787829790.216312:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787829790.216315:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085909248:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787829790.216320:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829790.216329:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829790.216344:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f61c0 x1874672085909888/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829790.216351:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787829790.216362:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829790.216366:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829790.216368:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787829790.216373:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829790/real 1787829790] req@ffff9e35da6f70c0 x1874672085909248/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829815 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829790.216377:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3225 00000100:00080000:1.0:1787829790.216384:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829790.216392:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829790.216399:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:0.0:1787829790.216401:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f70c0 x1874672085909248/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829815 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829790.216415:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1787829790.216416:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4f00 x1874672085910016/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829790.216424:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829790.216434:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085909248:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787829790.216437:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829790.216446:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829790.216450:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829790.216455:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3225 00000100:00080000:1.0:1787829790.216460:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829790.216465:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829790.216470:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:0.0:1787829790.216482:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085909376:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829790.216485:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6580 x1874672085910144/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829790.216518:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829790.216534:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829790/real 1787829790] req@ffff9e35da6f4b40 x1874672085909376/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829815 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829790.216546:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4b40 x1874672085909376/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829815 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829790.216554:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829790.216558:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829790.216560:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085909376:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829790.216566:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085909504:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829790.216587:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829790.216597:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829790/real 1787829790] req@ffff9e35da6f5e00 x1874672085909504/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829815 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829790.216607:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5e00 x1874672085909504/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829815 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829790.216615:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829790.216618:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829790.216621:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085909504:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829790.216626:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085909632:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829790.216645:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829790.216656:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829790/real 1787829790] req@ffff9e35da6f5a40 x1874672085909632/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829815 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829790.216668:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5a40 x1874672085909632/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829815 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829790.216676:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829790.216679:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829790.216682:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085909632:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829790.216688:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085909760:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829790.216707:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829790.216720:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829790/real 1787829790] req@ffff9e35da6f7840 x1874672085909760/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829815 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829790.216751:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7840 x1874672085909760/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829815 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829790.216760:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829790.216764:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829790.216767:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085909760:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829790.216773:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085909888:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829790.216795:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829790.216842:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829790/real 1787829790] req@ffff9e35da6f61c0 x1874672085909888/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829815 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829790.216854:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f61c0 x1874672085909888/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829815 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829790.216870:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829790.216874:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829790.216877:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085909888:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829790.216883:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085910016:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829790.216906:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829790.216917:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829790/real 1787829790] req@ffff9e35da6f4f00 x1874672085910016/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829815 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829790.216930:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4f00 x1874672085910016/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829815 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829790.216940:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829790.216944:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829790.216947:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085910016:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829790.216974:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085910144:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829790.216994:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829790.217007:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829790/real 1787829790] req@ffff9e35da6f6580 x1874672085910144/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829815 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829790.217018:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6580 x1874672085910144/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829815 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829790.217026:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829790.217030:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829790.217033:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085910144:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829790.343906:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829790.343915:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829790.343920:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829790.343938:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829791.367934:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829791.367940:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829791.367989:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829791.368039:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829792.391909:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829792.391919:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829792.391929:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787829792.391986:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829793.415949:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829793.415949:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829793.415979:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:0.0:1787829793.416188:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829794.439902:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829794.439904:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829794.439958:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787829794.439971:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829795.335949:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829795.335957:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829795.335960:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829795.335970:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3230 00000100:00080000:2.0:1787829795.335979:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829795.335985:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787829795.336020:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c000 x1874672085910272/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829795.336053:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829795.336057:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829795.336058:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829795.336062:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3230 00000100:00080000:2.0:1787829795.336065:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829795.336068:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829795.336077:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829795.336090:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cb40 x1874672085910400/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829795.336099:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829795.336101:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829795.336105:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787829795.336106:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085910272:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1787829795.336110:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3230 00000100:00080000:2.0:1787829795.336116:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829795.336121:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829795.336127:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829795.336140:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498fc00 x1874672085910528/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829795.336151:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829795.336154:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829795.336156:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829795.336161:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3230 00000100:00080000:2.0:1787829795.336166:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1787829795.336169:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1787829795.336175:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829795.336183:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:2.0:1787829795.336199:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498d680 x1874672085910656/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829795.336208:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829795/real 1787829795] req@ffff9e35c498c000 x1874672085910272/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829820 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829795.336213:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829795.336223:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829795.336226:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829795.336232:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3230 00000100:00100000:0.0:1787829795.336234:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c000 x1874672085910272/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829820 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829795.336237:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829795.336245:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:0.0:1787829795.336250:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:2.0:1787829795.336252:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00080000:0.0:1787829795.336255:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829795.336260:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085910272:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1787829795.336268:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e940 x1874672085910784/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829795.336275:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829795.336277:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829795.336279:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829795.336283:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3230 00000100:00100000:0.0:1787829795.336286:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085910400:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1787829795.336288:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829795.336295:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829795.336299:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829795.336309:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498de00 x1874672085910912/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829795.336316:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1787829795.336324:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829795.336327:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829795.336328:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829795.336331:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3230 00000100:00080000:2.0:1787829795.336335:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787829795.336336:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829795/real 1787829795] req@ffff9e35c498cb40 x1874672085910400/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829820 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829795.336340:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829795.336346:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829795.336356:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c000 x1874672085911040/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829795.336365:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498cb40 x1874672085910400/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829820 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829795.336366:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829795.336373:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829795.336376:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829795.336385:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3230 00000100:00080000:2.0:1787829795.336391:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829795.336394:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829795.336394:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829795.336398:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00080000:0.0:1787829795.336403:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829795.336407:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498ed00 x1874672085911168/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829795.336413:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085910400:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1787829795.336417:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829795.336419:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829795.336420:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829795.336427:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3230 00000100:00080000:2.0:1787829795.336429:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829795.336434:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787829795.336435:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085910528:192.168.203.141@tcp:38:kworker.0 00000008:00100000:2.0:1787829795.336438:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:2.0:1787829795.336450:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e580 x1874672085911296/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829795.336469:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829795.336488:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829795/real 1787829795] req@ffff9e35c498fc00 x1874672085910528/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829820 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829795.336502:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498fc00 x1874672085910528/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829820 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829795.336511:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829795.336515:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829795.336518:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085910528:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829795.336524:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085910656:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829795.336540:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829795.336548:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829795/real 1787829795] req@ffff9e35c498d680 x1874672085910656/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829820 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829795.336556:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498d680 x1874672085910656/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829820 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829795.336562:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829795.336564:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829795.336566:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085910656:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829795.336570:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085910784:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829795.336581:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829795.336587:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829795/real 1787829795] req@ffff9e35c498e940 x1874672085910784/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829820 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829795.336594:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e940 x1874672085910784/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829820 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829795.336600:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829795.336601:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829795.336603:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085910784:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829795.336620:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085910912:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829795.336631:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829795.336637:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829795/real 1787829795] req@ffff9e35c498de00 x1874672085910912/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829820 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829795.336645:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498de00 x1874672085910912/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829820 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829795.336650:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829795.336652:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829795.336654:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085910912:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829795.336658:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085911040:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829795.336667:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829795.336673:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829795/real 1787829795] req@ffff9e35c498c000 x1874672085911040/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829820 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829795.336685:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c000 x1874672085911040/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829820 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829795.336707:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829795.336709:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829795.336712:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085911040:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829795.336716:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085911168:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829795.336730:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829795.336737:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829795/real 1787829795] req@ffff9e35c498ed00 x1874672085911168/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829820 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829795.336745:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498ed00 x1874672085911168/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829820 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829795.336750:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829795.336753:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829795.336755:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085911168:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829795.336758:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085911296:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829795.336767:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829795.336772:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829795/real 1787829795] req@ffff9e35c498e580 x1874672085911296/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829820 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829795.336779:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e580 x1874672085911296/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829820 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829795.336793:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829795.336847:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829795.336850:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085911296:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829795.463892:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787829795.463915:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829795.463966:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829795.463966:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829796.487957:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:0.0:1787829796.488113:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829796.488114:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829796.488228:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829797.511907:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829797.511943:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787829797.511957:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829797.511976:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829798.535876:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829798.535876:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829798.535880:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787829798.535947:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829799.559926:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829799.559956:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829799.559974:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829799.559975:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829800.455919:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829800.455931:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787829800.455935:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829800.455948:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3235 00000100:00080000:3.0:1787829800.455962:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829800.455972:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787829800.456040:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499a40 x1874672085911424/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829800.456072:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829800.456075:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829800.456077:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829800.456082:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3235 00000100:00080000:3.0:1787829800.456087:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829800.456092:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829800.456112:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829800.456113:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085911424:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1787829800.456129:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498000 x1874672085911552/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829800.456142:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829800.456146:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787829800.456148:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829800.456153:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3235 00000100:00080000:3.0:1787829800.456157:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829800.456161:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829800.456167:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829800.456179:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649ad00 x1874672085911680/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829800.456188:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1787829800.456191:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829800.456196:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787829800.456198:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829800.456204:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3235 00000100:00080000:3.0:1787829800.456209:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829800.456214:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829800.456219:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:3.0:1787829800.456234:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c64983c0 x1874672085911808/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829800.456237:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829800/real 1787829800] req@ffff9e35c6499a40 x1874672085911424/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829825 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829800.456270:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829800.456273:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787829800.456276:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787829800.456278:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499a40 x1874672085911424/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829825 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829800.456282:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3235 00000100:00080000:3.0:1787829800.456288:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829800.456294:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829800.456301:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00080000:0.0:1787829800.456306:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1787829800.456312:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c64992c0 x1874672085911936/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829800.456315:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:3.0:1787829800.456326:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787829800.456327:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085911424:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1787829800.456332:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829800.456336:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829800.456341:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3235 00000100:00080000:3.0:1787829800.456343:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829800.456345:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829800.456348:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829800.456355:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b480 x1874672085912064/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829800.456363:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787829800.456365:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085911552:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1787829800.456367:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787829800.456370:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829800.456376:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3235 00000100:00080000:3.0:1787829800.456379:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829800.456382:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829800.456386:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829800.456394:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498780 x1874672085912192/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829800.456416:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829800.456420:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787829800.456422:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1787829800.456427:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1787829800.456431:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3235 00000100:00080000:3.0:1787829800.456437:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829800.456442:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829800.456448:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:0.0:1787829800.456452:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829800/real 1787829800] req@ffff9e35c6498000 x1874672085911552/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829825 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829800.456464:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498f00 x1874672085912320/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829800.456474:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498000 x1874672085911552/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829825 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829800.456476:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829800.456486:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787829800.456490:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829800.456500:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3235 00000100:00080000:0.0:1787829800.456502:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829800.456505:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829800.456510:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:3.0:1787829800.456513:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829800.456520:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:0.0:1787829800.456521:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085911552:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829800.456534:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085911680:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787829800.456537:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499a40 x1874672085912448/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829800.456560:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829800.456572:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829800/real 1787829800] req@ffff9e35c649ad00 x1874672085911680/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829825 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829800.456584:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649ad00 x1874672085911680/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829825 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829800.456592:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829800.456595:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829800.456598:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085911680:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829800.456602:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c64983c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085911808:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829800.456616:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829800.456624:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829800/real 1787829800] req@ffff9e35c64983c0 x1874672085911808/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829825 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829800.456633:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c64983c0 x1874672085911808/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829825 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829800.456650:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829800.456671:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829800.456676:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c64983c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085911808:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829800.456682:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c64992c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085911936:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829800.456716:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829800.456723:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829800/real 1787829800] req@ffff9e35c64992c0 x1874672085911936/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829825 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829800.456732:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c64992c0 x1874672085911936/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829825 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829800.456738:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829800.456740:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829800.456742:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c64992c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085911936:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829800.456756:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085912064:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829800.456767:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829800.456774:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829800/real 1787829800] req@ffff9e35c649b480 x1874672085912064/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829825 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829800.456781:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b480 x1874672085912064/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829825 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829800.456787:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829800.456789:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829800.456790:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085912064:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829800.456794:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085912192:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829800.456804:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829800.456810:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829800/real 1787829800] req@ffff9e35c6498780 x1874672085912192/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829825 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829800.456837:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498780 x1874672085912192/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829825 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829800.456843:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829800.456845:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829800.456847:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085912192:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829800.456850:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085912320:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829800.456866:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829800.456878:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829800/real 1787829800] req@ffff9e35c6498f00 x1874672085912320/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829825 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829800.456892:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498f00 x1874672085912320/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829825 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829800.456901:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829800.456905:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829800.456908:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085912320:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829800.456915:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085912448:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829800.456931:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829800.456939:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829800/real 1787829800] req@ffff9e35c6499a40 x1874672085912448/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829825 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829800.456946:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499a40 x1874672085912448/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829825 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829800.456953:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829800.456955:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829800.456957:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085912448:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829800.583920:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829800.583922:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829800.583948:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787829800.583976:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829801.607870:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829801.607880:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829801.607902:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:0.0:1787829801.607953:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829802.631908:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829802.631914:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829802.631937:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787829802.631948:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829803.655913:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829803.655929:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787829803.655952:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829803.655977:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829804.679921:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:3.0:1787829804.679921:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829804.679938:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:0.0:1787829804.680015:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829805.575909:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829805.575917:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829805.575920:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829805.575929:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3240 00000100:00080000:2.0:1787829805.575938:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829805.575944:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787829805.575977:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cf00 x1874672085912576/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829805.575999:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829805.576002:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829805.576007:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829805.576010:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3240 00000100:00080000:2.0:1787829805.576012:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829805.576014:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829805.576023:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829805.576032:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e1c0 x1874672085912704/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829805.576050:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085912576:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1787829805.576053:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829805.576057:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829805.576059:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829805.576062:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3240 00000100:00080000:2.0:1787829805.576064:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829805.576067:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829805.576071:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829805.576080:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f480 x1874672085912832/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829805.576086:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829805.576089:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829805.576090:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829805.576093:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3240 00000100:00080000:2.0:1787829805.576095:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829805.576097:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829805.576101:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:2.0:1787829805.576108:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c780 x1874672085912960/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829805.576111:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1787829805.576119:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829805.576123:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829805.576124:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829805.576128:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3240 00000100:00080000:2.0:1787829805.576132:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829805.576135:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829805.576138:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:2.0:1787829805.576152:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c3c0 x1874672085913088/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829805.576160:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829805/real 1787829805] req@ffff9e35c498cf00 x1874672085912576/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829830 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829805.576162:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829805.576171:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829805.576175:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829805.576182:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3240 00000100:00080000:2.0:1787829805.576186:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787829805.576189:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498cf00 x1874672085912576/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829830 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829805.576191:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829805.576198:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1787829805.576211:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:2.0:1787829805.576211:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f840 x1874672085913216/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829805.576221:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:2.0:1787829805.576224:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829805.576229:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829805.576232:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787829805.576233:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085912576:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1787829805.576236:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3240 00000100:00080000:2.0:1787829805.576240:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829805.576245:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829805.576248:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829805.576255:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f0c0 x1874672085913344/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829805.576266:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085912704:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1787829805.576269:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829805.576273:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829805.576274:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829805.576278:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3240 00000100:00080000:2.0:1787829805.576281:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829805.576284:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829805.576288:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000800:00000100:0.0:1787829805.576293:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829805.576298:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cf00 x1874672085913472/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829805.576306:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787829805.576308:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829805/real 1787829805] req@ffff9e35c498e1c0 x1874672085912704/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829830 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829805.576309:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829805.576312:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829805.576320:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3240 00000100:00080000:2.0:1787829805.576325:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829805.576331:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829805.576338:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:0.0:1787829805.576344:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e1c0 x1874672085912704/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829830 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829805.576347:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e580 x1874672085913600/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829805.576377:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829805.576383:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829805.576389:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085912704:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829805.576397:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085912832:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829805.576416:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829805.576427:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829805/real 1787829805] req@ffff9e35c498f480 x1874672085912832/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829830 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829805.576440:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f480 x1874672085912832/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829830 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829805.576449:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829805.576453:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829805.576456:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085912832:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829805.576463:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085912960:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829805.576483:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829805.576491:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829805/real 1787829805] req@ffff9e35c498c780 x1874672085912960/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829830 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829805.576499:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c780 x1874672085912960/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829830 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829805.576510:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829805.576512:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829805.576514:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085912960:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829805.576518:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085913088:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829805.576528:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829805.576534:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829805/real 1787829805] req@ffff9e35c498c3c0 x1874672085913088/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829830 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829805.576541:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c3c0 x1874672085913088/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829830 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829805.576547:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829805.576549:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829805.576551:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085913088:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829805.576554:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085913216:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829805.576564:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829805.576570:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829805/real 1787829805] req@ffff9e35c498f840 x1874672085913216/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829830 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829805.576577:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f840 x1874672085913216/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829830 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829805.576582:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829805.576584:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829805.576586:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085913216:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829805.576604:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085913344:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829805.576614:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829805.576642:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829805/real 1787829805] req@ffff9e35c498f0c0 x1874672085913344/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829830 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829805.576651:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f0c0 x1874672085913344/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829830 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829805.576656:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829805.576658:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829805.576660:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085913344:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829805.576664:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085913472:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829805.576680:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829805.576691:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829805/real 1787829805] req@ffff9e35c498cf00 x1874672085913472/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829830 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829805.576703:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498cf00 x1874672085913472/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829830 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829805.576712:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829805.576716:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829805.576719:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085913472:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829805.576725:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085913600:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829805.576740:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829805.576748:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829805/real 1787829805] req@ffff9e35c498e580 x1874672085913600/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829830 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829805.576755:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e580 x1874672085913600/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829830 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829805.576761:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829805.576763:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829805.576765:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085913600:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829805.703901:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829805.703901:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:3.0:1787829805.703949:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787829805.703950:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829806.727916:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829806.727917:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:3.0:1787829806.727917:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787829806.728004:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829807.751866:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829807.751890:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829807.751909:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787829807.751940:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829808.775898:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829808.775911:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829808.775928:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787829808.775983:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829809.799939:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829809.799993:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787829809.800009:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829809.800013:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00080000:2.0:1787829810.695891:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829810.695899:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829810.695902:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829810.695915:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3246 00000100:00080000:2.0:1787829810.695924:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829810.695931:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787829810.695964:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498ed00 x1874672085913728/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829810.695987:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829810.695990:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829810.695991:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829810.695994:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3246 00000100:00080000:2.0:1787829810.695997:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829810.695999:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829810.696007:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829810.696016:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c000 x1874672085913856/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829810.696023:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829810.696026:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829810.696029:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829810.696032:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3246 00000100:00080000:2.0:1787829810.696033:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829810.696036:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829810.696039:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829810.696045:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498de00 x1874672085913984/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829810.696054:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829810.696057:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829810.696058:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829810.696065:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3246 00000100:00100000:0.0:1787829810.696067:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085913728:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1787829810.696068:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829810.696078:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829810.696081:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:2.0:1787829810.696090:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e940 x1874672085914112/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829810.696096:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829810.696098:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829810.696100:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829810.696102:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3246 00000100:00080000:2.0:1787829810.696104:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829810.696106:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829810.696110:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:2.0:1787829810.696119:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498d680 x1874672085914240/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829810.696124:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829810.696126:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829810.696128:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829810.696132:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3246 00000100:00080000:2.0:1787829810.696135:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829810.696137:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829810.696143:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829810.696149:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498fc00 x1874672085914368/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829810.696155:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829810.696157:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829810.696158:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829810.696161:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3246 00000100:00080000:2.0:1787829810.696163:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829810.696165:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829810.696168:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829810.696187:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cb40 x1874672085914496/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829810.696189:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1787829810.696199:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829810.696202:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829810.696205:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829810.696208:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3246 00000100:00080000:2.0:1787829810.696212:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829810.696215:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829810.696219:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:2.0:1787829810.696231:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e580 x1874672085914624/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829810.696238:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829810.696241:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829810.696243:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829810.696247:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3246 00000100:00080000:2.0:1787829810.696250:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787829810.696252:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829810/real 1787829810] req@ffff9e35c498ed00 x1874672085913728/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829835 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829810.696258:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829810.696265:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:2.0:1787829810.696284:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cf00 x1874672085914752/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829810.696292:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498ed00 x1874672085913728/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829835 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829810.696314:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829810.696319:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829810.696326:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085913728:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1787829810.696336:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085913856:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829810.696358:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829810.696367:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829810/real 1787829810] req@ffff9e35c498c000 x1874672085913856/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829835 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829810.696375:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c000 x1874672085913856/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829835 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829810.696383:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829810.696386:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829810.696388:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085913856:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829810.696414:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085913984:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829810.696426:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829810.696433:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829810/real 1787829810] req@ffff9e35c498de00 x1874672085913984/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829835 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829810.696440:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498de00 x1874672085913984/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829835 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829810.696446:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829810.696448:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829810.696450:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085913984:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829810.696454:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085914112:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829810.696464:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829810.696470:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829810/real 1787829810] req@ffff9e35c498e940 x1874672085914112/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829835 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829810.696478:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e940 x1874672085914112/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829835 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829810.696483:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829810.696485:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829810.696488:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085914112:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829810.696492:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085914240:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829810.696502:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829810.696507:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829810/real 1787829810] req@ffff9e35c498d680 x1874672085914240/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829835 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829810.696515:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498d680 x1874672085914240/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829835 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829810.696520:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829810.696522:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829810.696524:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085914240:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829810.696528:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085914368:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829810.696538:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829810.696544:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829810/real 1787829810] req@ffff9e35c498fc00 x1874672085914368/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829835 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829810.696552:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498fc00 x1874672085914368/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829835 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829810.696557:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829810.696559:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829810.696561:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085914368:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829810.696564:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085914496:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829810.696574:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829810.696580:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829810/real 1787829810] req@ffff9e35c498cb40 x1874672085914496/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829835 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829810.696607:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498cb40 x1874672085914496/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829835 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829810.696613:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829810.696615:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829810.696617:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085914496:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829810.696622:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085914624:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829810.696638:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829810.696645:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829810/real 1787829810] req@ffff9e35c498e580 x1874672085914624/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829835 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829810.696653:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e580 x1874672085914624/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829835 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829810.696658:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829810.696660:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829810.696662:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085914624:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829810.696666:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085914752:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829810.696677:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829810.696682:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829810/real 1787829810] req@ffff9e35c498cf00 x1874672085914752/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829835 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829810.696689:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498cf00 x1874672085914752/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829835 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829810.696696:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829810.696698:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829810.696699:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085914752:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829810.823926:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829810.823954:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829810.823972:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829810.823987:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829811.847964:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829811.847964:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829811.847964:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787829811.848029:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829812.871933:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829812.871975:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787829812.872002:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829812.872003:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829813.895913:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829813.895914:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829813.895916:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829813.895996:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829814.919903:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829814.919903:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829814.919926:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787829814.919966:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829815.815893:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829815.815901:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829815.815904:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829815.815913:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3251 00000100:00080000:1.0:1787829815.815924:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829815.815929:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787829815.815963:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4780 x1874672085914880/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829815.815984:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829815.815987:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829815.815989:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829815.815992:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3251 00000100:00080000:1.0:1787829815.815994:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829815.816003:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829815.816014:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829815.816026:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6d00 x1874672085915008/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829815.816034:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829815.816039:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:0.0:1787829815.816041:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085914880:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787829815.816041:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829815.816046:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3251 00000100:00080000:1.0:1787829815.816051:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829815.816053:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829815.816057:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829815.816067:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6940 x1874672085915136/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829815.816074:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829815.816077:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829815.816078:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829815.816081:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3251 00000100:00080000:1.0:1787829815.816095:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829815.816097:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829815.816101:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000800:00000100:0.0:1787829815.816105:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829815.816112:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4000 x1874672085915264/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829815.816120:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829815.816123:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829815.816124:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829815.816127:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3251 00000100:00080000:1.0:1787829815.816130:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829815.816132:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829815.816137:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:0.0:1787829815.816144:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829815/real 1787829815] req@ffff9e35da6f4780 x1874672085914880/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829840 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829815.816147:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7480 x1874672085915392/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829815.816169:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829815.816172:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829815.816173:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787829815.816176:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4780 x1874672085914880/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829840 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829815.816180:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3251 00000100:00080000:1.0:1787829815.816185:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829815.816190:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829815.816198:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1787829815.816199:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829815.816205:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829815.816211:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6580 x1874672085915520/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829815.816212:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085914880:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787829815.816225:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829815.816227:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829815.816229:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829815.816232:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3251 00000100:00080000:1.0:1787829815.816233:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829815.816236:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787829815.816241:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085915008:192.168.203.141@tcp:38:kworker.0 00000008:00100000:1.0:1787829815.816242:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829815.816261:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4f00 x1874672085915648/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829815.816268:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787829815.816282:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787829815.816283:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829815/real 1787829815] req@ffff9e35da6f6d00 x1874672085915008/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829840 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829815.816293:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829815.816297:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787829815.816303:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6d00 x1874672085915008/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829840 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829815.816305:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3251 00000100:00080000:1.0:1787829815.816311:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829815.816318:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:0.0:1787829815.816319:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829815.816325:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000008:00100000:1.0:1787829815.816326:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:0.0:1787829815.816329:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085915008:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829815.816336:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085915136:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829815.816345:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f61c0 x1874672085915776/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829815.816354:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787829815.816364:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787829815.816366:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829815/real 1787829815] req@ffff9e35da6f6940 x1874672085915136/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829840 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829815.816373:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829815.816379:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787829815.816384:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6940 x1874672085915136/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829840 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829815.816387:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3251 00000100:00080000:1.0:1787829815.816393:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829815.816401:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829815.816402:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:0.0:1787829815.816410:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000008:00100000:1.0:1787829815.816411:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:0.0:1787829815.816416:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085915136:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829815.816422:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085915264:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829815.816427:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7840 x1874672085915904/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829815.816442:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829815.816451:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829815/real 1787829815] req@ffff9e35da6f4000 x1874672085915264/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829840 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829815.816460:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4000 x1874672085915264/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829840 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829815.816466:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829815.816468:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829815.816470:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085915264:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829815.816474:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085915392:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829815.816484:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829815.816489:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829815/real 1787829815] req@ffff9e35da6f7480 x1874672085915392/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829840 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829815.816496:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7480 x1874672085915392/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829840 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829815.816502:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829815.816503:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829815.816505:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085915392:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829815.816509:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085915520:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829815.816527:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829815.816538:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829815/real 1787829815] req@ffff9e35da6f6580 x1874672085915520/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829840 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829815.816570:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6580 x1874672085915520/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829840 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829815.816579:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829815.816583:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829815.816586:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085915520:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829815.816610:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085915648:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829815.816631:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829815.816639:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829815/real 1787829815] req@ffff9e35da6f4f00 x1874672085915648/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829840 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829815.816647:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4f00 x1874672085915648/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829840 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829815.816653:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829815.816655:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829815.816657:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085915648:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829815.816660:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085915776:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829815.816671:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829815.816676:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829815/real 1787829815] req@ffff9e35da6f61c0 x1874672085915776/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829840 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829815.816683:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f61c0 x1874672085915776/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829840 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829815.816689:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829815.816690:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829815.816692:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085915776:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829815.816695:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085915904:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829815.816704:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829815.816710:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829815/real 1787829815] req@ffff9e35da6f7840 x1874672085915904/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829840 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829815.816717:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7840 x1874672085915904/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829840 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829815.816722:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829815.816724:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829815.816726:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085915904:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829815.943912:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829815.943913:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829815.943916:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787829815.943945:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829816.967911:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829816.967911:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829816.967955:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829816.967974:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829817.991906:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829817.991906:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829817.991935:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787829817.991947:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829819.015898:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829819.015898:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829819.015899:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787829819.015936:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829820.039900:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829820.039900:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829820.039916:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787829820.039941:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829820.935940:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829820.935958:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787829820.935964:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829820.935975:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3256 00000100:00080000:3.0:1787829820.935986:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829820.935993:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787829820.936039:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a580 x1874672085916032/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829820.936071:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829820.936076:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829820.936077:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829820.936083:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3256 00000100:00080000:3.0:1787829820.936087:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829820.936091:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829820.936102:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829820.936110:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085916032:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1787829820.936116:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499e00 x1874672085916160/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829820.936128:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829820.936132:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787829820.936134:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829820.936138:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3256 00000100:00080000:3.0:1787829820.936142:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829820.936146:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829820.936151:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829820.936176:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b0c0 x1874672085916288/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787829820.936182:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829820.936218:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829820/real 1787829820] req@ffff9e35c649a580 x1874672085916032/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829845 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829820.936238:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a580 x1874672085916032/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829845 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829820.936250:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829820.936255:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829820.936259:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085916032:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1787829820.936272:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829820.936277:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787829820.936279:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787829820.936287:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085916160:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787829820.936318:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829820.936329:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829820/real 1787829820] req@ffff9e35c6499e00 x1874672085916160/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829845 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829820.936338:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499e00 x1874672085916160/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829845 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829820.936344:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829820.936346:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829820.936349:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085916160:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829820.936354:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085916288:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787829820.936376:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829820.936390:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829820/real 1787829820] req@ffff9e35c649b0c0 x1874672085916288/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829845 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829820.936396:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3256 00000100:00080000:3.0:1787829820.936403:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829820.936411:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829820.936419:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:1.0:1787829820.936425:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b0c0 x1874672085916288/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829845 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829820.936434:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1787829820.936435:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a940 x1874672085916416/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829820.936442:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829820.936446:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085916288:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1787829820.936498:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829820.936503:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787829820.936505:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829820.936522:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3256 00000100:00080000:3.0:1787829820.936526:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829820.936530:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829820.936537:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:0.0:1787829820.936543:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085916416:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787829820.936550:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499680 x1874672085916544/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829820.936560:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829820.936568:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829820.936570:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829820.936573:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3256 00000100:00080000:3.0:1787829820.936576:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829820.936579:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829820.936583:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1787829820.936586:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787829820.936595:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b840 x1874672085916672/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829820.936602:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829820.936605:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787829820.936607:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787829820.936610:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829820/real 1787829820] req@ffff9e35c649a940 x1874672085916416/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829845 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829820.936611:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3256 00000100:00080000:3.0:1787829820.936616:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829820.936620:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829820.936626:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829820.936636:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649bc00 x1874672085916800/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829820.936643:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a940 x1874672085916416/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829845 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829820.936655:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829820.936662:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829820.936663:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787829820.936668:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829820.936673:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:3.0:1787829820.936678:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3256 00000100:00100000:0.0:1787829820.936683:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085916416:192.168.203.141@tcp:8:kworker.0 00000100:00080000:3.0:1787829820.936685:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829820.936695:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829820.936701:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:0.0:1787829820.936708:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085916544:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787829820.936718:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498b40 x1874672085916928/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829820.936731:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1787829820.936737:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1787829820.936744:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787829820.936747:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829820.936752:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3256 00000100:00100000:0.0:1787829820.936758:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829820/real 1787829820] req@ffff9e35c6499680 x1874672085916544/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829845 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829820.936759:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829820.936768:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829820.936783:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:0.0:1787829820.936784:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499680 x1874672085916544/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829845 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829820.936826:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829820.936830:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787829820.936837:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a1c0 x1874672085917056/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829820.936838:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085916544:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829820.936854:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085916672:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829820.936880:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829820.936893:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829820/real 1787829820] req@ffff9e35c649b840 x1874672085916672/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829845 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829820.936906:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b840 x1874672085916672/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829845 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829820.936915:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829820.936918:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829820.936922:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085916672:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829820.936927:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085916800:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829820.936946:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829820.936957:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829820/real 1787829820] req@ffff9e35c649bc00 x1874672085916800/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829845 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829820.936969:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649bc00 x1874672085916800/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829845 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829820.936977:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829820.936980:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829820.936984:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085916800:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829820.936999:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085916928:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829820.937021:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829820.937032:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829820/real 1787829820] req@ffff9e35c6498b40 x1874672085916928/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829845 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829820.937044:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498b40 x1874672085916928/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829845 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829820.937053:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829820.937056:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829820.937059:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085916928:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829820.937065:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085917056:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829820.937085:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829820.937097:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829820/real 1787829820] req@ffff9e35c649a1c0 x1874672085917056/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829845 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829820.937110:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a1c0 x1874672085917056/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829845 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829820.937119:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829820.937123:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829820.937126:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085917056:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829821.063877:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829821.063878:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829821.063888:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787829821.063912:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829822.087902:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829822.087907:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829822.087912:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787829822.087965:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829823.111884:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829823.111884:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829823.111889:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787829823.111971:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829824.135900:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829824.135900:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829824.135911:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787829824.135941:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829825.159870:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829825.159870:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829825.159877:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787829825.159972:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829826.055870:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829826.055876:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829826.055879:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829826.055887:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3261 00000100:00080000:1.0:1787829826.055895:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829826.055900:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787829826.055931:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5a40 x1874672085917184/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829826.055957:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829826.055961:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829826.055963:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829826.055967:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3261 00000100:00080000:1.0:1787829826.055971:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829826.055975:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829826.055983:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829826.055991:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085917184:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1787829826.055995:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5e00 x1874672085917312/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829826.056003:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829826.056006:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829826.056007:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829826.056009:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3261 00000100:00080000:1.0:1787829826.056012:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829826.056014:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829826.056016:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829826.056023:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4b40 x1874672085917440/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829826.056028:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829826.056030:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829826.056031:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829826.056034:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3261 00000100:00080000:1.0:1787829826.056036:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829826.056038:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829826.056041:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000800:00000100:0.0:1787829826.056045:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829826.056048:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f70c0 x1874672085917568/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829826.056055:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829826.056058:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829826.056059:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829826.056062:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3261 00000100:00080000:1.0:1787829826.056064:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829826.056066:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829826.056068:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:1.0:1787829826.056077:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f43c0 x1874672085917696/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829826.056080:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829826/real 1787829826] req@ffff9e35da6f5a40 x1874672085917184/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829851 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829826.056090:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829826.056094:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829826.056096:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829826.056099:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3261 00000100:00100000:0.0:1787829826.056102:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5a40 x1874672085917184/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829851 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829826.056103:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829826.056109:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829826.056115:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1787829826.056118:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829826.056123:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829826.056126:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085917184:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1787829826.056131:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7840 x1874672085917824/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829826.056137:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829826.056140:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829826.056141:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829826.056143:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3261 00000100:00080000:1.0:1787829826.056146:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787829826.056147:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085917312:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787829826.056150:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829826.056156:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829826.056165:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f61c0 x1874672085917952/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829826.056170:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787829826.056178:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829826.056180:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829826.056181:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787829826.056185:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829826/real 1787829826] req@ffff9e35da6f5e00 x1874672085917312/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829851 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829826.056187:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3261 00000100:00080000:1.0:1787829826.056193:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829826.056197:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829826.056200:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:1.0:1787829826.056208:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4f00 x1874672085918080/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829826.056211:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5e00 x1874672085917312/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829851 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829826.056223:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829826.056226:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829826.056227:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829826.056229:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829826.056232:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:1.0:1787829826.056236:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3261 00000100:00100000:0.0:1787829826.056239:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085917312:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787829826.056240:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829826.056246:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787829826.056249:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085917440:192.168.203.141@tcp:38:kworker.0 00000008:00100000:1.0:1787829826.056251:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:1.0:1787829826.056262:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6580 x1874672085918208/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829826.056270:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829826.056278:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829826/real 1787829826] req@ffff9e35da6f4b40 x1874672085917440/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829851 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829826.056287:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4b40 x1874672085917440/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829851 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829826.056302:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829826.056305:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829826.056308:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085917440:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829826.056313:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085917568:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829826.056324:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829826.056330:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829826/real 1787829826] req@ffff9e35da6f70c0 x1874672085917568/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829851 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829826.056336:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f70c0 x1874672085917568/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829851 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829826.056341:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829826.056342:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829826.056344:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085917568:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829826.056347:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085917696:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829826.056355:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829826.056359:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829826/real 1787829826] req@ffff9e35da6f43c0 x1874672085917696/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829851 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829826.056370:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f43c0 x1874672085917696/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829851 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829826.056375:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829826.056377:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829826.056379:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085917696:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829826.056382:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085917824:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829826.056390:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829826.056396:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829826/real 1787829826] req@ffff9e35da6f7840 x1874672085917824/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829851 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829826.056402:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7840 x1874672085917824/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829851 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829826.056406:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829826.056408:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829826.056409:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085917824:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829826.056423:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085917952:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829826.056432:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829826.056437:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829826/real 1787829826] req@ffff9e35da6f61c0 x1874672085917952/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829851 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829826.056443:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f61c0 x1874672085917952/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829851 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829826.056451:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829826.056453:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829826.056455:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085917952:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829826.056458:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085918080:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829826.056466:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829826.056471:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829826/real 1787829826] req@ffff9e35da6f4f00 x1874672085918080/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829851 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829826.056491:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4f00 x1874672085918080/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829851 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829826.056497:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829826.056499:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829826.056500:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085918080:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829826.056504:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085918208:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829826.056516:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829826.056522:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829826/real 1787829826] req@ffff9e35da6f6580 x1874672085918208/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829851 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829826.056528:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6580 x1874672085918208/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829851 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829826.056533:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829826.056535:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829826.056537:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085918208:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829826.183893:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829826.183893:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829826.183904:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829826.183936:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829827.207858:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829827.207886:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829827.207916:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787829827.208049:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829828.231905:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829828.231905:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829828.231906:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787829828.231954:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829829.255881:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829829.255882:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829829.255922:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829829.255924:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829830.279938:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829830.279971:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829830.279993:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787829830.280057:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829831.175890:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829831.175897:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829831.175900:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829831.175908:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3266 00000100:00080000:1.0:1787829831.175916:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829831.175926:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787829831.175955:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7480 x1874672085918336/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829831.175978:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829831.175981:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829831.175982:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829831.175984:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3266 00000100:00080000:1.0:1787829831.175986:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829831.175989:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829831.175997:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829831.176005:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4000 x1874672085918464/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829831.176023:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829831.176026:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829831.176028:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829831.176032:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3266 00000100:00080000:1.0:1787829831.176035:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829831.176041:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787829831.176042:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085918336:192.168.203.141@tcp:250:kworker.0 00000008:00100000:1.0:1787829831.176047:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829831.176059:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6940 x1874672085918592/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829831.176072:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829831.176075:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829831.176076:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829831.176079:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3266 00000100:00080000:1.0:1787829831.176095:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829831.176098:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829831.176101:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000800:00000100:0.0:1787829831.176107:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829831.176112:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6d00 x1874672085918720/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829831.176119:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829831.176122:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829831.176123:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829831.176126:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3266 00000100:00080000:1.0:1787829831.176128:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829831.176134:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829831.176139:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:0.0:1787829831.176150:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829831/real 1787829831] req@ffff9e35da6f7480 x1874672085918336/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829856 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829831.176152:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4780 x1874672085918848/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829831.176182:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829831.176184:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829831.176186:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787829831.176191:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7480 x1874672085918336/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829856 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829831.176191:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3266 00000100:00080000:1.0:1787829831.176193:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829831.176199:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829831.176206:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1787829831.176210:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829831.176217:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829831.176218:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6580 x1874672085918976/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829831.176227:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085918336:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787829831.176235:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787829831.176238:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085918464:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787829831.176239:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829831.176244:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829831.176250:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3266 00000100:00080000:1.0:1787829831.176253:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829831.176257:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829831.176262:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1787829831.176269:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829831.176276:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4f00 x1874672085919104/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829831.176285:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829831/real 1787829831] req@ffff9e35da6f4000 x1874672085918464/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829856 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829831.176288:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829831.176292:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829831.176295:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829831.176304:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3266 00000100:00080000:1.0:1787829831.176309:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787829831.176311:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4000 x1874672085918464/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829856 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829831.176316:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829831.176325:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00080000:0.0:1787829831.176328:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829831.176334:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829831.176338:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085918464:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829831.176341:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f61c0 x1874672085919232/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829831.176349:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829831.176350:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829831.176352:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829831.176354:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3266 00000100:00080000:1.0:1787829831.176357:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829831.176360:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829831.176363:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:0.0:1787829831.176368:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085918592:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829831.176372:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7840 x1874672085919360/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829831.176403:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829831.176420:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829831/real 1787829831] req@ffff9e35da6f6940 x1874672085918592/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829856 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829831.176431:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6940 x1874672085918592/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829856 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829831.176454:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829831.176458:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829831.176461:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085918592:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829831.176467:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085918720:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829831.176487:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829831.176497:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829831/real 1787829831] req@ffff9e35da6f6d00 x1874672085918720/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829856 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829831.176508:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6d00 x1874672085918720/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829856 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829831.176516:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829831.176518:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829831.176522:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085918720:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829831.176527:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085918848:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829831.176545:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829831.176554:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829831/real 1787829831] req@ffff9e35da6f4780 x1874672085918848/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829856 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829831.176564:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4780 x1874672085918848/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829856 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829831.176570:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829831.176573:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829831.176576:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085918848:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829831.176581:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085918976:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829831.176596:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829831.176604:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829831/real 1787829831] req@ffff9e35da6f6580 x1874672085918976/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829856 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829831.176614:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6580 x1874672085918976/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829856 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829831.176622:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829831.176625:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829831.176628:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085918976:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829831.176633:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085919104:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829831.176645:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829831.176654:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829831/real 1787829831] req@ffff9e35da6f4f00 x1874672085919104/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829856 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829831.176671:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4f00 x1874672085919104/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829856 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829831.176678:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829831.176680:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829831.176683:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085919104:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829831.176688:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085919232:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829831.176703:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829831.176711:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829831/real 1787829831] req@ffff9e35da6f61c0 x1874672085919232/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829856 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829831.176722:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f61c0 x1874672085919232/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829856 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829831.176729:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829831.176732:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829831.176735:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085919232:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829831.176755:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085919360:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829831.176770:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829831.176779:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829831/real 1787829831] req@ffff9e35da6f7840 x1874672085919360/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829856 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829831.176788:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7840 x1874672085919360/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829856 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829831.176824:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829831.176827:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829831.176830:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085919360:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829831.303883:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829831.303893:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829831.303919:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787829831.303934:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829832.327886:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829832.327886:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829832.327895:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787829832.327931:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829833.351919:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829833.351920:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787829833.351979:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829833.352070:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829834.375890:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829834.375890:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829834.375914:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787829834.375934:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829835.399933:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829835.399934:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829835.399934:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:0.0:1787829835.400022:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829836.295923:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829836.295933:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829836.295938:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829836.295948:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3271 00000100:00080000:1.0:1787829836.295959:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829836.295966:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787829836.296001:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f43c0 x1874672085919488/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829836.296032:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829836.296036:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829836.296038:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829836.296043:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3271 00000100:00080000:1.0:1787829836.296047:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829836.296050:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829836.296061:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829836.296074:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f70c0 x1874672085919616/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829836.296089:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085919488:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787829836.296101:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829836.296106:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829836.296108:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829836.296117:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3271 00000100:00080000:1.0:1787829836.296121:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829836.296125:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829836.296130:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829836.296142:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4b40 x1874672085919744/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829836.296152:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829836.296155:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829836.296158:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1787829836.296159:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787829836.296166:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3271 00000100:00080000:1.0:1787829836.296171:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829836.296176:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829836.296182:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:1.0:1787829836.296194:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5e00 x1874672085919872/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829836.296204:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829836.296210:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:0.0:1787829836.296211:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829836/real 1787829836] req@ffff9e35da6f43c0 x1874672085919488/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829861 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829836.296213:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829836.296222:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3271 00000100:00080000:1.0:1787829836.296227:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829836.296232:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829836.296237:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:0.0:1787829836.296240:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f43c0 x1874672085919488/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829861 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829836.296252:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5a40 x1874672085920000/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829836.296253:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829836.296262:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829836.296270:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085919488:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787829836.296271:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829836.296279:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829836.296281:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829836.296286:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3271 00000100:00080000:1.0:1787829836.296290:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787829836.296299:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085919616:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787829836.296305:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829836.296316:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1787829836.296327:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829836.296331:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f43c0 x1874672085920128/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829836.296342:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787829836.296344:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829836/real 1787829836] req@ffff9e35da6f70c0 x1874672085919616/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829861 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829836.296347:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829836.296351:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829836.296360:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3271 00000100:00080000:1.0:1787829836.296367:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829836.296374:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829836.296381:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829836.296385:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f70c0 x1874672085919616/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829861 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829836.296395:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7840 x1874672085920256/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829836.296398:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829836.296420:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:1.0:1787829836.296426:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787829836.296427:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085919616:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787829836.296433:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829836.296436:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787829836.296439:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085919744:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787829836.296444:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3271 00000100:00080000:1.0:1787829836.296449:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829836.296454:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829836.296459:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000800:00000100:0.0:1787829836.296467:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829836.296474:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f61c0 x1874672085920384/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829836.296483:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829836/real 1787829836] req@ffff9e35da6f4b40 x1874672085919744/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829861 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829836.296487:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829836.296495:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829836.296498:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829836.296505:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3271 00000100:00100000:0.0:1787829836.296508:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4b40 x1874672085919744/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829861 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829836.296510:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829836.296519:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:0.0:1787829836.296523:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000008:00100000:1.0:1787829836.296526:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00080000:0.0:1787829836.296530:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829836.296536:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085919744:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829836.296542:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4f00 x1874672085920512/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829836.296545:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085919872:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829836.296574:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829836.296589:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829836/real 1787829836] req@ffff9e35da6f5e00 x1874672085919872/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829861 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829836.296602:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5e00 x1874672085919872/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829861 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829836.296617:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829836.296620:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829836.296623:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085919872:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829836.296630:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085920000:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829836.296648:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829836.296659:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829836/real 1787829836] req@ffff9e35da6f5a40 x1874672085920000/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829861 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829836.296669:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5a40 x1874672085920000/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829861 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829836.296677:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829836.296681:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829836.296684:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085920000:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829836.296706:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085920128:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829836.296726:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829836.296736:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829836/real 1787829836] req@ffff9e35da6f43c0 x1874672085920128/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829861 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829836.296747:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f43c0 x1874672085920128/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829861 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829836.296755:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829836.296759:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829836.296762:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085920128:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829836.296768:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085920256:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829836.296785:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829836.296795:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829836/real 1787829836] req@ffff9e35da6f7840 x1874672085920256/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829861 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829836.296845:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7840 x1874672085920256/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829861 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829836.296853:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829836.296855:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829836.296858:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085920256:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829836.296864:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085920384:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829836.296884:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829836.296895:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829836/real 1787829836] req@ffff9e35da6f61c0 x1874672085920384/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829861 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829836.296905:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f61c0 x1874672085920384/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829861 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829836.296913:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829836.296916:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829836.296919:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085920384:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829836.296924:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085920512:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829836.296939:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829836.296950:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829836/real 1787829836] req@ffff9e35da6f4f00 x1874672085920512/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829861 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829836.296961:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4f00 x1874672085920512/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829861 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829836.296968:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829836.296971:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829836.296974:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085920512:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829836.423896:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829836.423897:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829836.423897:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:0.0:1787829836.423950:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829837.447932:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829837.447936:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829837.447936:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787829837.447990:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829838.471877:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829838.471882:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829838.471916:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:1.0:1787829838.471928:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829839.495911:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829839.495911:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829839.495941:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:0.0:1787829839.495948:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829840.519872:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829840.519872:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829840.519888:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:1.0:1787829840.519918:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829841.415877:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829841.415885:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829841.415889:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829841.415898:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3276 00000100:00080000:1.0:1787829841.415907:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829841.415913:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787829841.415941:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6580 x1874672085920640/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829841.415963:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829841.415966:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829841.415967:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829841.415970:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3276 00000100:00080000:1.0:1787829841.415972:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829841.415975:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829841.415983:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829841.415991:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4780 x1874672085920768/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829841.415995:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085920640:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787829841.416008:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829841.416010:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829841.416011:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829841.416014:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3276 00000100:00080000:1.0:1787829841.416016:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829841.416018:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829841.416022:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829841.416030:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6d00 x1874672085920896/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829841.416035:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829841.416038:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829841.416039:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829841.416044:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3276 00000100:00080000:1.0:1787829841.416047:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829841.416050:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829841.416055:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000800:00000100:0.0:1787829841.416060:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829841.416067:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6940 x1874672085921024/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829841.416078:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829841.416081:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829841.416083:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829841.416088:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3276 00000100:00080000:1.0:1787829841.416092:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829841.416095:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787829841.416099:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829841/real 1787829841] req@ffff9e35da6f6580 x1874672085920640/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829866 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1787829841.416099:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:1.0:1787829841.416111:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4000 x1874672085921152/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829841.416118:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6580 x1874672085920640/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829866 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829841.416120:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829841.416127:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829841.416130:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829841.416137:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3276 00000100:00080000:0.0:1787829841.416138:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829841.416140:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829841.416145:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:1.0:1787829841.416145:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829841.416150:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829841.416151:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085920640:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1787829841.416159:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7480 x1874672085921280/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829841.416165:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829841.416167:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829841.416168:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829841.416171:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3276 00000100:00080000:1.0:1787829841.416173:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829841.416175:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829841.416177:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829841.416185:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085920768:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829841.416186:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6580 x1874672085921408/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829841.416206:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829841.416210:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000100:0.0:1787829841.416211:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787829841.416214:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829841.416221:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3276 00000100:00100000:0.0:1787829841.416224:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829841/real 1787829841] req@ffff9e35da6f4780 x1874672085920768/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829866 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829841.416226:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829841.416235:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787829841.416238:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4780 x1874672085920768/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829866 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1787829841.416241:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00080000:0.0:1787829841.416249:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829841.416252:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829841.416258:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085920768:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829841.416258:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4f00 x1874672085921536/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829841.416266:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085920896:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787829841.416269:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829841.416273:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829841.416275:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829841.416278:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3276 00000100:00080000:1.0:1787829841.416282:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1787829841.416285:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787829841.416289:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829841.416293:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:0.0:1787829841.416296:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829841/real 1787829841] req@ffff9e35da6f6d00 x1874672085920896/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829866 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829841.416303:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f61c0 x1874672085921664/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829841.416309:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6d00 x1874672085920896/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829866 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829841.416319:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829841.416321:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829841.416323:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085920896:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829841.416328:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085921024:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829841.416338:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829841.416344:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829841/real 1787829841] req@ffff9e35da6f6940 x1874672085921024/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829866 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829841.416350:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6940 x1874672085921024/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829866 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829841.416355:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829841.416356:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829841.416358:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085921024:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829841.416361:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085921152:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829841.416384:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829841.416391:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829841/real 1787829841] req@ffff9e35da6f4000 x1874672085921152/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829866 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829841.416398:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4000 x1874672085921152/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829866 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829841.416403:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829841.416404:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829841.416406:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085921152:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829841.416409:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085921280:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829841.416420:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829841.416425:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829841/real 1787829841] req@ffff9e35da6f7480 x1874672085921280/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829866 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829841.416431:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7480 x1874672085921280/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829866 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829841.416436:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829841.416437:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829841.416439:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085921280:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829841.416453:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085921408:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829841.416461:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829841.416466:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829841/real 1787829841] req@ffff9e35da6f6580 x1874672085921408/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829866 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829841.416472:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6580 x1874672085921408/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829866 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829841.416476:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829841.416478:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829841.416479:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085921408:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829841.416482:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085921536:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829841.416490:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829841.416495:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829841/real 1787829841] req@ffff9e35da6f4f00 x1874672085921536/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829866 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829841.416502:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4f00 x1874672085921536/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829866 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829841.416506:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829841.416508:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829841.416509:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085921536:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829841.416512:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085921664:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829841.416520:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829841.416526:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829841/real 1787829841] req@ffff9e35da6f61c0 x1874672085921664/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829866 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829841.416532:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f61c0 x1874672085921664/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829866 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829841.416537:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829841.416539:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829841.416540:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085921664:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829841.543866:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829841.543871:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:1.0:1787829841.543901:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829841.543911:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829842.567869:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829842.567878:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829842.567883:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:0.0:1787829842.567932:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829843.591892:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829843.591892:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829843.591902:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787829843.591960:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829844.615883:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829844.615884:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829844.615885:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:1.0:1787829844.615916:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829845.639903:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829845.639933:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829845.639952:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:0.0:1787829845.639971:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829846.535898:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829846.535906:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829846.535910:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829846.535920:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3281 00000100:00080000:1.0:1787829846.535932:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829846.535940:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787829846.535980:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7840 x1874672085921792/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829846.536020:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829846.536024:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829846.536026:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829846.536030:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3281 00000100:00080000:1.0:1787829846.536035:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829846.536039:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829846.536050:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829846.536066:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f43c0 x1874672085921920/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829846.536075:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085921792:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787829846.536083:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829846.536088:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829846.536090:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829846.536095:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3281 00000100:00080000:1.0:1787829846.536098:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829846.536101:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829846.536105:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829846.536115:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5a40 x1874672085922048/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829846.536123:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829846.536125:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829846.536126:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829846.536129:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3281 00000100:00080000:1.0:1787829846.536132:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1787829846.536136:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787829846.536140:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829846.536145:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:1.0:1787829846.536153:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5e00 x1874672085922176/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829846.536160:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829846.536162:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829846.536163:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829846.536167:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3281 00000100:00080000:1.0:1787829846.536170:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829846.536175:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787829846.536176:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829846/real 1787829846] req@ffff9e35da6f7840 x1874672085921792/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829871 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1787829846.536178:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:1.0:1787829846.536192:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4b40 x1874672085922304/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829846.536197:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7840 x1874672085921792/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829871 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829846.536202:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829846.536209:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829846.536213:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829846.536217:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829846.536218:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3281 00000100:00080000:1.0:1787829846.536224:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829846.536225:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:1.0:1787829846.536229:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787829846.536232:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085921792:192.168.203.141@tcp:250:kworker.0 00000008:00100000:1.0:1787829846.536234:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829846.536245:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f70c0 x1874672085922432/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829846.536251:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829846.536253:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829846.536254:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787829846.536258:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085921920:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787829846.536260:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3281 00000100:00080000:1.0:1787829846.536266:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829846.536268:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829846.536272:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829846.536281:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7840 x1874672085922560/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829846.536285:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787829846.536295:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829846.536298:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:0.0:1787829846.536302:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829846/real 1787829846] req@ffff9e35da6f43c0 x1874672085921920/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829871 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829846.536302:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829846.536311:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3281 00000100:00080000:1.0:1787829846.536317:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829846.536321:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787829846.536324:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f43c0 x1874672085921920/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829871 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1787829846.536326:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00080000:0.0:1787829846.536355:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1787829846.536359:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f61c0 x1874672085922688/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829846.536362:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829846.536370:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085921920:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787829846.536379:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787829846.536383:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085922048:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787829846.536384:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829846.536388:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829846.536393:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3281 00000100:00080000:1.0:1787829846.536396:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829846.536399:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829846.536403:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000800:00000100:0.0:1787829846.536408:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829846.536414:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4f00 x1874672085922816/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829846.536421:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829846/real 1787829846] req@ffff9e35da6f5a40 x1874672085922048/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829871 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829846.536432:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5a40 x1874672085922048/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829871 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829846.536439:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829846.536442:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829846.536444:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085922048:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829846.536449:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085922176:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829846.536460:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829846.536467:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829846/real 1787829846] req@ffff9e35da6f5e00 x1874672085922176/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829871 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829846.536479:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5e00 x1874672085922176/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829871 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829846.536484:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829846.536486:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829846.536488:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085922176:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829846.536492:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085922304:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829846.536502:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829846.536508:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829846/real 1787829846] req@ffff9e35da6f4b40 x1874672085922304/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829871 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829846.536515:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4b40 x1874672085922304/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829871 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829846.536521:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829846.536523:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829846.536525:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085922304:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829846.536528:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085922432:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829846.536537:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829846.536542:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829846/real 1787829846] req@ffff9e35da6f70c0 x1874672085922432/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829871 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829846.536549:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f70c0 x1874672085922432/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829871 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829846.536559:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829846.536561:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829846.536563:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085922432:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829846.536589:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085922560:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829846.536600:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829846.536607:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829846/real 1787829846] req@ffff9e35da6f7840 x1874672085922560/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829871 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829846.536614:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7840 x1874672085922560/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829871 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829846.536620:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829846.536621:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829846.536623:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085922560:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829846.536627:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085922688:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829846.536637:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829846.536643:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829846/real 1787829846] req@ffff9e35da6f61c0 x1874672085922688/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829871 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829846.536650:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f61c0 x1874672085922688/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829871 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829846.536655:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829846.536657:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829846.536659:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085922688:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829846.536662:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085922816:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829846.536671:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829846.536676:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829846/real 1787829846] req@ffff9e35da6f4f00 x1874672085922816/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829871 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829846.536683:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4f00 x1874672085922816/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829871 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829846.536688:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829846.536689:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829846.536691:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085922816:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829846.663875:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829846.663892:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829846.663918:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787829846.663931:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829847.687934:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829847.687952:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829847.687976:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787829847.688016:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829848.711876:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829848.711880:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829848.711903:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787829848.711940:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829849.735909:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829849.735910:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829849.735935:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829849.735955:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829850.759941:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829850.759941:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829850.759983:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787829850.760019:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829851.655933:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829851.655942:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787829851.655945:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829851.655954:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3287 00000100:00080000:0.0:1787829851.655963:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829851.655970:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787829851.656004:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7192c0 x1874672085922944/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829851.656027:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829851.656031:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829851.656032:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829851.656035:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3287 00000100:00080000:0.0:1787829851.656038:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829851.656042:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829851.656051:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829851.656064:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085922944:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1787829851.656065:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718780 x1874672085923072/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829851.656079:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829851.656081:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787829851.656083:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829851.656087:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3287 00000100:00080000:0.0:1787829851.656089:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829851.656091:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829851.656094:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829851.656103:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a940 x1874672085923200/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829851.656109:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829851.656112:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829851.656113:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829851.656117:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3287 00000100:00080000:0.0:1787829851.656119:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829851.656121:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829851.656126:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:0.0:1787829851.656133:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718f00 x1874672085923328/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829851.656140:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829851.656142:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829851.656144:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829851.656146:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3287 00000100:00080000:0.0:1787829851.656148:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829851.656151:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829851.656154:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:0.0:1787829851.656162:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7183c0 x1874672085923456/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829851.656170:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787829851.656172:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085923072:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787829851.656175:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829851.656179:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829851.656183:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3287 00000100:00080000:0.0:1787829851.656186:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829851.656189:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829851.656193:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829851.656202:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da719a40 x1874672085923584/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829851.656207:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085923200:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787829851.656216:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829851.656219:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787829851.656221:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829851.656226:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3287 00000100:00080000:0.0:1787829851.656229:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829851.656234:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787829851.656234:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085923328:192.168.203.141@tcp:8:kworker.0 00000008:00100000:0.0:1787829851.656242:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829851.656254:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a580 x1874672085923712/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829851.656262:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787829851.656265:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085923456:192.168.203.141@tcp:8:kworker.0 00000100:00080000:0.0:1787829851.656267:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829851.656271:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829851.656277:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3287 00000100:00080000:0.0:1787829851.656281:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829851.656283:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829851.656288:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:1.0:1787829851.656295:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085923584:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829851.656314:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718000 x1874672085923840/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829851.656322:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829851.656324:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829851.656325:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829851.656329:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3287 00000100:00080000:0.0:1787829851.656333:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1787829851.656338:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085923712:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787829851.656339:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829851.656348:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:1.0:1787829851.656363:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085923840:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829851.656364:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718b40 x1874672085923968/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829851.656389:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085923968:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829851.783878:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829851.783888:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829851.783895:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787829851.783937:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829852.807931:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829852.807932:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829852.807937:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829852.807967:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829853.831900:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829853.831923:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829853.831942:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787829853.832033:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1787829854.728023:0:2397:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.141:988 00000400:00000100:2.0:1787829854.728078:0:2397:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.141@tcp at host 192.168.203.141 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1787829854.728093:0:2397:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:2.0:1787829854.728147:0:2397:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000800:00000100:2.0:1787829854.728162:0:2397:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:2.0:1787829854.728170:0:2397:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:2.0:1787829854.728178:0:2397:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:2.0:1787829854.728185:0:2397:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:2.0:1787829854.728193:0:2397:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:2.0:1787829854.728201:0:2397:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:2.0:1787829854.728209:0:2397:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:1.0:1787829854.728309:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829851/real 1787829854] req@ffff9e35da718b40 x1874672085923968/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829876 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829854.728337:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718b40 x1874672085923968/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829876 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829854.728351:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829854.728357:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829854.728362:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085923968:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829854.728370:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829851/real 1787829854] req@ffff9e35da71a580 x1874672085923712/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829876 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829854.728384:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672085923712/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829876 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829854.728393:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829854.728396:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829854.728401:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085923712:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829854.728408:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829851/real 1787829854] req@ffff9e35da718000 x1874672085923840/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829876 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829854.728421:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718000 x1874672085923840/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829876 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829854.728430:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829854.728433:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829854.728436:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085923840:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829854.728442:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829851/real 1787829854] req@ffff9e35da7183c0 x1874672085923456/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829876 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829854.728455:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672085923456/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829876 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829854.728464:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829854.728467:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829854.728471:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085923456:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829854.728478:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829851/real 1787829854] req@ffff9e35da719a40 x1874672085923584/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829876 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829854.728491:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672085923584/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829876 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829854.728499:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829854.728503:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829854.728507:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085923584:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829854.728515:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829851/real 1787829854] req@ffff9e35da718780 x1874672085923072/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829876 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829854.728528:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718780 x1874672085923072/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829876 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829854.728537:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829854.728541:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829854.728545:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085923072:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829854.728552:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829851/real 1787829854] req@ffff9e35da71a940 x1874672085923200/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829876 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829854.728564:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672085923200/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829876 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829854.728572:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829854.728575:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829854.728578:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085923200:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829854.728584:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829851/real 1787829854] req@ffff9e35da718f00 x1874672085923328/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829876 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829854.728595:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672085923328/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829876 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829854.728603:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829854.728606:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829854.728610:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085923328:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829854.728617:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829851/real 1787829854] req@ffff9e35da7192c0 x1874672085922944/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829876 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829854.728630:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7192c0 x1874672085922944/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829876 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829854.728638:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829854.728641:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829854.728644:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085922944:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1787829854.855908:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829854.855908:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829854.855930:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:0.0:1787829854.855993:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829855.879895:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829855.879910:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829855.879929:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829855.879947:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00080000:3.0:1787829856.775947:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829856.775959:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787829856.775964:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829856.775975:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3292 00000100:00080000:3.0:1787829856.775988:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829856.775996:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787829856.776041:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a1c0 x1874672085924096/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829856.776072:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829856.776076:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829856.776078:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829856.776083:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3292 00000100:00080000:3.0:1787829856.776087:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829856.776092:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829856.776104:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829856.776122:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498b40 x1874672085924224/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829856.776136:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829856.776139:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787829856.776143:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787829856.776146:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085924096:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1787829856.776151:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3292 00000100:00080000:3.0:1787829856.776159:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829856.776165:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829856.776171:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829856.776187:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649bc00 x1874672085924352/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829856.776198:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829856.776203:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787829856.776206:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829856.776210:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3292 00000100:00080000:3.0:1787829856.776215:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829856.776221:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829856.776227:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000800:00000100:1.0:1787829856.776238:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787829856.776260:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b840 x1874672085924480/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829856.776284:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829856.776288:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787829856.776290:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829856.776296:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3292 00000100:00080000:3.0:1787829856.776301:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1787829856.776309:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829856/real 1787829856] req@ffff9e35c649a1c0 x1874672085924096/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829881 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829856.776309:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829856.776318:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:3.0:1787829856.776340:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499680 x1874672085924608/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829856.776345:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a1c0 x1874672085924096/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829881 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829856.776365:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829856.776370:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829856.776371:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829856.776376:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829856.776381:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:3.0:1787829856.776387:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3292 00000100:00100000:1.0:1787829856.776391:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085924096:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1787829856.776394:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829856.776403:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829856.776411:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829856.776426:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a940 x1874672085924736/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829856.776433:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085924224:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1787829856.776446:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829856.776451:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787829856.776454:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829856.776460:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3292 00000100:00080000:3.0:1787829856.776464:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829856.776469:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:1.0:1787829856.776475:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:3.0:1787829856.776476:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829856.776495:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b0c0 x1874672085924864/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829856.776498:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829856/real 1787829856] req@ffff9e35c6498b40 x1874672085924224/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829881 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829856.776516:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829856.776522:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:1.0:1787829856.776526:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498b40 x1874672085924224/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829881 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829856.776527:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829856.776538:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3292 00000100:00080000:3.0:1787829856.776546:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829856.776549:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829856.776554:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:1.0:1787829856.776558:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000008:00100000:3.0:1787829856.776564:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:1.0:1787829856.776565:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085924224:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829856.776578:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085924352:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787829856.776585:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499e00 x1874672085924992/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829856.776597:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829856.776601:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787829856.776604:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1787829856.776608:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1787829856.776613:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3292 00000100:00080000:3.0:1787829856.776618:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829856.776623:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829856.776630:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:1.0:1787829856.776634:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829856/real 1787829856] req@ffff9e35c649bc00 x1874672085924352/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829881 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829856.776647:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a580 x1874672085925120/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829856.776653:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649bc00 x1874672085924352/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829881 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829856.776673:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829856.776678:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829856.776683:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085924352:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829856.776693:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085924480:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829856.776724:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829856.776736:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829856/real 1787829856] req@ffff9e35c649b840 x1874672085924480/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829881 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829856.776749:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b840 x1874672085924480/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829881 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829856.776765:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829856.776769:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829856.776772:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085924480:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829856.776778:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085924608:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829856.776855:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829856.776875:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829856/real 1787829856] req@ffff9e35c6499680 x1874672085924608/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829881 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829856.776888:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499680 x1874672085924608/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829881 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829856.776897:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829856.776900:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829856.776904:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085924608:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829856.776931:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085924736:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787829856.776956:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829856.776968:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829856/real 1787829856] req@ffff9e35c649a940 x1874672085924736/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829881 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829856.776982:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a940 x1874672085924736/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829881 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829856.776991:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829856.776994:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829856.776998:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085924736:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829856.777005:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085924864:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787829856.777025:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829856.777037:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829856/real 1787829856] req@ffff9e35c649b0c0 x1874672085924864/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829881 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829856.777049:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b0c0 x1874672085924864/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829881 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829856.777056:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829856.777060:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829856.777063:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085924864:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829856.777070:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085924992:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829856.777090:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829856.777102:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829856/real 1787829856] req@ffff9e35c6499e00 x1874672085924992/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829881 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829856.777115:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499e00 x1874672085924992/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829881 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829856.777125:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829856.777128:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829856.777131:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085924992:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829856.777138:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085925120:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829856.777167:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829856.777182:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829856/real 1787829856] req@ffff9e35c649a580 x1874672085925120/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829881 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829856.777195:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a580 x1874672085925120/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829881 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829856.777204:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829856.777208:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829856.777211:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085925120:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829856.903939:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829856.903939:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829856.903949:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829856.903989:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829857.927885:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829857.927906:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:0.0:1787829857.927938:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829857.927965:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829858.951907:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829858.951915:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829858.951943:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829858.951952:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829859.975897:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829859.975917:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829859.975936:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829859.975966:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829860.999937:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829860.999984:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829860.999984:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787829860.999992:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829861.895916:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829861.895930:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829861.895935:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829861.895945:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3297 00000100:00080000:1.0:1787829861.895956:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829861.895962:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787829861.896005:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6580 x1874672085925248/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829861.896035:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829861.896038:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829861.896040:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829861.896043:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3297 00000100:00080000:1.0:1787829861.896045:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829861.896048:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829861.896057:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829861.896067:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7480 x1874672085925376/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829861.896076:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829861.896079:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829861.896080:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829861.896085:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3297 00000100:00100000:0.0:1787829861.896086:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085925248:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787829861.896088:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829861.896094:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829861.896098:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829861.896107:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4000 x1874672085925504/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829861.896113:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829861.896116:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829861.896117:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829861.896120:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3297 00000100:00080000:1.0:1787829861.896121:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829861.896124:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829861.896127:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:1.0:1787829861.896135:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6940 x1874672085925632/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829861.896141:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829861.896144:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829861.896145:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829861.896149:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3297 00000100:00080000:1.0:1787829861.896153:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829861.896155:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829861.896159:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000800:00000100:0.0:1787829861.896163:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829861.896169:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6d00 x1874672085925760/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829861.896176:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829861.896179:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829861.896181:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829861.896185:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3297 00000100:00080000:1.0:1787829861.896187:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829861.896189:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829861.896193:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829861.896201:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4780 x1874672085925888/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829861.896207:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787829861.896211:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829861/real 1787829861] req@ffff9e35da6f6580 x1874672085925248/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829886 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829861.896212:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829861.896216:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829861.896223:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3297 00000100:00080000:1.0:1787829861.896244:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829861.896249:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829861.896254:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829861.896257:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6580 x1874672085925248/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829886 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829861.896269:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4f00 x1874672085926016/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829861.896270:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829861.896279:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:1.0:1787829861.896287:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787829861.896288:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085925248:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787829861.896295:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829861.896298:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829861.896302:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3297 00000100:00080000:1.0:1787829861.896305:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829861.896308:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829861.896311:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:1.0:1787829861.896318:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f61c0 x1874672085926144/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829861.896322:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085925376:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787829861.896335:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829861.896337:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829861.896339:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829861.896343:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3297 00000100:00080000:1.0:1787829861.896348:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1787829861.896352:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787829861.896353:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829861.896360:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:0.0:1787829861.896369:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829861/real 1787829861] req@ffff9e35da6f7480 x1874672085925376/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829886 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829861.896373:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7840 x1874672085926272/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829861.896399:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7480 x1874672085925376/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829886 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829861.896407:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829861.896410:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829861.896413:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085925376:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829861.896417:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085925504:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829861.896431:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829861.896438:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829861/real 1787829861] req@ffff9e35da6f4000 x1874672085925504/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829886 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829861.896446:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4000 x1874672085925504/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829886 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829861.896451:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829861.896453:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829861.896455:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085925504:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829861.896459:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085925632:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829861.896472:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829861.896482:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829861/real 1787829861] req@ffff9e35da6f6940 x1874672085925632/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829886 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829861.896494:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6940 x1874672085925632/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829886 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829861.896504:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829861.896507:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829861.896511:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085925632:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829861.896517:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085925760:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829861.896537:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829861.896545:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829861/real 1787829861] req@ffff9e35da6f6d00 x1874672085925760/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829886 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829861.896552:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6d00 x1874672085925760/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829886 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829861.896559:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829861.896561:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829861.896563:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085925760:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829861.896566:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085925888:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829861.896577:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829861.896583:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829861/real 1787829861] req@ffff9e35da6f4780 x1874672085925888/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829886 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829861.896590:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4780 x1874672085925888/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829886 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829861.896595:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829861.896597:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829861.896600:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085925888:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829861.896604:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085926016:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829861.896614:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829861.896620:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829861/real 1787829861] req@ffff9e35da6f4f00 x1874672085926016/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829886 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829861.896627:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4f00 x1874672085926016/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829886 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829861.896633:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829861.896634:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829861.896636:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085926016:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829861.896653:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085926144:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829861.896663:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829861.896670:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829861/real 1787829861] req@ffff9e35da6f61c0 x1874672085926144/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829886 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829861.896677:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f61c0 x1874672085926144/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829886 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829861.896683:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829861.896685:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829861.896686:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085926144:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829861.896690:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085926272:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829861.896699:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829861.896706:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829861/real 1787829861] req@ffff9e35da6f7840 x1874672085926272/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829886 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829861.896713:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7840 x1874672085926272/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829886 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829861.896718:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829861.896720:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829861.896722:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085926272:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829862.023862:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829862.023874:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829862.023952:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829862.024041:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829863.047888:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829863.047889:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829863.047959:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829863.047964:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829864.071879:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829864.071882:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829864.071896:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787829864.071935:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829865.095897:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829865.095898:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829865.095937:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829865.095941:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829866.119879:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829866.119905:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829866.119908:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829866.119924:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00080000:2.0:1787829867.015930:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829867.015939:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829867.015943:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829867.015952:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3302 00000100:00080000:2.0:1787829867.015961:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829867.015966:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787829867.015999:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f0c0 x1874672085926400/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829867.016020:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829867.016024:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829867.016025:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829867.016028:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3302 00000100:00080000:2.0:1787829867.016031:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829867.016033:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829867.016043:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829867.016051:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f840 x1874672085926528/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829867.016058:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829867.016062:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829867.016063:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829867.016066:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3302 00000100:00080000:2.0:1787829867.016068:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829867.016073:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787829867.016074:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085926400:192.168.203.141@tcp:250:kworker.0 00000008:00100000:2.0:1787829867.016077:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829867.016087:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c3c0 x1874672085926656/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829867.016095:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829867.016097:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829867.016099:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829867.016101:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3302 00000100:00080000:2.0:1787829867.016104:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829867.016106:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829867.016110:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:2.0:1787829867.016117:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c780 x1874672085926784/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829867.016124:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829867.016127:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829867.016128:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829867.016131:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3302 00000100:00080000:2.0:1787829867.016134:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1787829867.016138:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1787829867.016141:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829867.016146:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:2.0:1787829867.016155:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f480 x1874672085926912/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829867.016162:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829867.016164:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829867.016166:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829867.016169:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3302 00000100:00080000:2.0:1787829867.016172:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829867.016174:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787829867.016179:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829867/real 1787829867] req@ffff9e35c498f0c0 x1874672085926400/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829892 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1787829867.016179:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829867.016204:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e1c0 x1874672085927040/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829867.016213:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787829867.016217:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f0c0 x1874672085926400/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829892 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829867.016218:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829867.016221:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829867.016230:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3302 00000100:00080000:0.0:1787829867.016234:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829867.016235:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829867.016242:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:0.0:1787829867.016243:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000008:00100000:2.0:1787829867.016249:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829867.016250:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085926400:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1787829867.016259:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085926528:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829867.016265:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cf00 x1874672085927168/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829867.016274:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829867.016276:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829867.016278:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829867.016280:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3302 00000100:00080000:2.0:1787829867.016284:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1787829867.016287:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1787829867.016292:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829867.016296:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:0.0:1787829867.016302:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829867/real 1787829867] req@ffff9e35c498f840 x1874672085926528/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829892 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829867.016308:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e580 x1874672085927296/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829867.016323:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787829867.016324:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f840 x1874672085926528/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829892 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829867.016331:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829867.016335:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829867.016344:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829867.016344:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3302 00000100:00080000:2.0:1787829867.016351:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829867.016352:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:2.0:1787829867.016356:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787829867.016359:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085926528:192.168.203.141@tcp:38:kworker.0 00000008:00100000:2.0:1787829867.016362:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:2.0:1787829867.016372:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cb40 x1874672085927424/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829867.016385:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085926656:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829867.016402:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829867.016409:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829867/real 1787829867] req@ffff9e35c498c3c0 x1874672085926656/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829892 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829867.016417:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c3c0 x1874672085926656/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829892 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829867.016423:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829867.016425:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829867.016428:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085926656:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829867.016432:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085926784:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829867.016442:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829867.016448:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829867/real 1787829867] req@ffff9e35c498c780 x1874672085926784/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829892 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829867.016461:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c780 x1874672085926784/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829892 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829867.016466:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829867.016468:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829867.016470:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085926784:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829867.016474:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085926912:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829867.016483:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829867.016489:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829867/real 1787829867] req@ffff9e35c498f480 x1874672085926912/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829892 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829867.016496:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f480 x1874672085926912/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829892 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829867.016502:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829867.016504:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829867.016505:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085926912:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829867.016510:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085927040:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829867.016528:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829867.016539:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829867/real 1787829867] req@ffff9e35c498e1c0 x1874672085927040/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829892 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829867.016552:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e1c0 x1874672085927040/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829892 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829867.016567:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829867.016570:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829867.016572:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085927040:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829867.016576:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085927168:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829867.016591:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829867.016599:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829867/real 1787829867] req@ffff9e35c498cf00 x1874672085927168/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829892 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829867.016606:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498cf00 x1874672085927168/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829892 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829867.016612:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829867.016614:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829867.016616:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085927168:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829867.016619:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085927296:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829867.016629:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829867.016634:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829867/real 1787829867] req@ffff9e35c498e580 x1874672085927296/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829892 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829867.016641:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e580 x1874672085927296/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829892 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829867.016647:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829867.016648:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829867.016651:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085927296:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829867.016654:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085927424:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829867.016663:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829867.016668:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829867/real 1787829867] req@ffff9e35c498cb40 x1874672085927424/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829892 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829867.016674:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498cb40 x1874672085927424/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829892 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829867.016679:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829867.016681:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829867.016683:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085927424:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829867.143904:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829867.143904:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829867.143917:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787829867.143942:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829868.167901:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829868.167901:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829868.167917:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829868.167934:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829869.191906:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829869.191906:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829869.191906:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829869.191964:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829870.215906:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829870.215908:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787829870.215938:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829870.215944:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829871.239894:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829871.239894:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829871.239901:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829871.239929:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829872.135885:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829872.135893:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829872.135897:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829872.135905:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3307 00000100:00080000:2.0:1787829872.135915:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829872.135921:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787829872.135955:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498fc00 x1874672085927552/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829872.135980:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829872.135983:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829872.135985:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829872.135989:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3307 00000100:00080000:2.0:1787829872.135993:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829872.135997:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829872.136008:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829872.136025:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498d680 x1874672085927680/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829872.136029:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085927552:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1787829872.136043:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829872.136046:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829872.136048:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829872.136052:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3307 00000100:00080000:2.0:1787829872.136055:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829872.136059:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829872.136063:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829872.136073:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e940 x1874672085927808/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829872.136079:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829872.136082:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829872.136083:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829872.136086:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3307 00000100:00080000:2.0:1787829872.136088:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1787829872.136093:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1787829872.136094:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829872.136101:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:2.0:1787829872.136110:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498de00 x1874672085927936/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829872.136116:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829872.136119:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829872.136120:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829872.136123:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3307 00000100:00080000:2.0:1787829872.136126:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829872.136128:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829872.136132:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:0.0:1787829872.136135:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829872/real 1787829872] req@ffff9e35c498fc00 x1874672085927552/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829897 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829872.136142:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c000 x1874672085928064/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829872.136178:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829872.136183:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:0.0:1787829872.136184:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498fc00 x1874672085927552/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829897 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829872.136185:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829872.136194:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3307 00000100:00080000:0.0:1787829872.136200:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829872.136200:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829872.136205:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:0.0:1787829872.136209:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000008:00100000:2.0:1787829872.136211:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829872.136217:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085927552:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1787829872.136223:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498ed00 x1874672085928192/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829872.136231:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829872.136233:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829872.136234:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829872.136237:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3307 00000100:00080000:2.0:1787829872.136239:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829872.136241:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829872.136244:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829872.136250:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085927680:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829872.136253:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498fc00 x1874672085928320/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829872.136264:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829872.136267:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829872.136268:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829872.136271:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3307 00000100:00080000:2.0:1787829872.136275:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829872.136277:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:0.0:1787829872.136282:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:2.0:1787829872.136286:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:0.0:1787829872.136295:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829872/real 1787829872] req@ffff9e35c498d680 x1874672085927680/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829897 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829872.136299:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cb40 x1874672085928448/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829872.136320:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787829872.136322:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498d680 x1874672085927680/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829897 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829872.136324:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829872.136328:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829872.136337:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3307 00000100:00080000:0.0:1787829872.136341:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829872.136341:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829872.136349:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:0.0:1787829872.136350:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000008:00100000:2.0:1787829872.136356:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:0.0:1787829872.136357:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085927680:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829872.136369:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e580 x1874672085928576/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829872.136370:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085927808:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829872.136391:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829872.136401:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829872/real 1787829872] req@ffff9e35c498e940 x1874672085927808/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829897 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829872.136410:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e940 x1874672085927808/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829897 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829872.136416:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829872.136427:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829872.136429:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085927808:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829872.136433:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085927936:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829872.136445:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829872.136452:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829872/real 1787829872] req@ffff9e35c498de00 x1874672085927936/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829897 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829872.136460:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498de00 x1874672085927936/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829897 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829872.136465:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829872.136467:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829872.136469:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085927936:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829872.136473:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085928064:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829872.136482:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829872.136487:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829872/real 1787829872] req@ffff9e35c498c000 x1874672085928064/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829897 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829872.136494:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c000 x1874672085928064/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829897 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829872.136500:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829872.136501:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829872.136504:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085928064:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829872.136509:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085928192:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829872.136526:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829872.136537:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829872/real 1787829872] req@ffff9e35c498ed00 x1874672085928192/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829897 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829872.136550:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498ed00 x1874672085928192/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829897 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829872.136560:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829872.136564:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829872.136567:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085928192:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829872.136592:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085928320:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829872.136613:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829872.136625:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829872/real 1787829872] req@ffff9e35c498fc00 x1874672085928320/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829897 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829872.136638:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498fc00 x1874672085928320/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829897 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829872.136648:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829872.136651:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829872.136654:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085928320:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829872.136659:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085928448:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829872.136681:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829872.136689:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829872/real 1787829872] req@ffff9e35c498cb40 x1874672085928448/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829897 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829872.136697:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498cb40 x1874672085928448/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829897 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829872.136703:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829872.136705:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829872.136707:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085928448:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829872.136711:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085928576:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829872.136721:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829872.136727:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829872/real 1787829872] req@ffff9e35c498e580 x1874672085928576/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829897 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829872.136735:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e580 x1874672085928576/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829897 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829872.136740:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829872.136742:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829872.136743:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085928576:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829872.263884:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829872.263885:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829872.263921:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829872.264002:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829873.287902:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829873.287930:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:1.0:1787829873.287934:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829873.287936:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829874.311898:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829874.311899:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829874.311924:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829874.311944:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829875.335874:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829875.335879:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829875.335895:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829875.335930:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829876.359916:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829876.359918:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829876.359918:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787829876.359963:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829877.255930:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829877.255939:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829877.255942:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829877.255951:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3312 00000100:00080000:1.0:1787829877.255961:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829877.255968:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787829877.256009:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f70c0 x1874672085928704/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829877.256034:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829877.256037:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829877.256039:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829877.256042:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3312 00000100:00080000:1.0:1787829877.256045:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829877.256049:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829877.256058:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829877.256071:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4b40 x1874672085928832/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829877.256081:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085928704:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787829877.256082:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829877.256088:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829877.256092:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829877.256095:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3312 00000100:00080000:1.0:1787829877.256098:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829877.256100:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829877.256103:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829877.256112:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5e00 x1874672085928960/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829877.256164:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787829877.256182:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829877.256187:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829877.256189:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829877.256196:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3312 00000100:00080000:1.0:1787829877.256201:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829877.256206:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787829877.256212:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829877/real 1787829877] req@ffff9e35da6f70c0 x1874672085928704/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829902 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1787829877.256215:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:1.0:1787829877.256234:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5a40 x1874672085929088/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829877.256241:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f70c0 x1874672085928704/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829902 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829877.256245:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829877.256253:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829877.256256:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829877.256261:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829877.256262:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3312 00000100:00080000:1.0:1787829877.256270:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829877.256272:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:1.0:1787829877.256275:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829877.256279:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:1.0:1787829877.256286:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f43c0 x1874672085929216/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829877.256293:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829877.256295:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829877.256296:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829877.256299:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3312 00000100:00080000:1.0:1787829877.256301:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829877.256303:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829877.256305:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829877.256311:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7840 x1874672085929344/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829877.256316:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829877.256318:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829877.256321:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787829877.256329:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085928704:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787829877.256329:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3312 00000100:00080000:1.0:1787829877.256332:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829877.256339:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829877.256345:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829877.256357:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f61c0 x1874672085929472/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829877.256363:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085928832:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787829877.256378:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829877.256383:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829877.256386:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1787829877.256399:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787829877.256401:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3312 00000100:00080000:1.0:1787829877.256409:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829877.256414:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829877.256420:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:0.0:1787829877.256421:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829877/real 1787829877] req@ffff9e35da6f4b40 x1874672085928832/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829902 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829877.256439:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4f00 x1874672085929600/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829877.256443:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4b40 x1874672085928832/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829902 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829877.256462:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829877.256466:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829877.256468:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829877.256472:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829877.256476:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:1.0:1787829877.256481:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3312 00000100:00100000:0.0:1787829877.256486:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085928832:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787829877.256487:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829877.256494:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829877.256500:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:0.0:1787829877.256502:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085928960:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829877.256512:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4780 x1874672085929728/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829877.256533:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829877.256551:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829877/real 1787829877] req@ffff9e35da6f5e00 x1874672085928960/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829902 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829877.256565:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5e00 x1874672085928960/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829902 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829877.256575:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829877.256579:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829877.256583:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085928960:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829877.256589:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085929088:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829877.256610:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829877.256622:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829877/real 1787829877] req@ffff9e35da6f5a40 x1874672085929088/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829902 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829877.256635:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5a40 x1874672085929088/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829902 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829877.256644:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829877.256648:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829877.256652:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085929088:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829877.256658:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085929216:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829877.256677:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829877.256690:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829877/real 1787829877] req@ffff9e35da6f43c0 x1874672085929216/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829902 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829877.256702:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f43c0 x1874672085929216/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829902 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829877.256712:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829877.256715:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829877.256719:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085929216:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829877.256725:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085929344:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829877.256744:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829877.256755:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829877/real 1787829877] req@ffff9e35da6f7840 x1874672085929344/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829902 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829877.256767:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7840 x1874672085929344/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829902 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829877.256775:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829877.256779:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829877.256782:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085929344:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829877.256841:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085929472:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829877.256861:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829877.256871:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829877/real 1787829877] req@ffff9e35da6f61c0 x1874672085929472/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829902 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829877.256883:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f61c0 x1874672085929472/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829902 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829877.256891:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829877.256894:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829877.256898:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085929472:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829877.256904:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085929600:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829877.256922:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829877.256931:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829877/real 1787829877] req@ffff9e35da6f4f00 x1874672085929600/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829902 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829877.256943:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4f00 x1874672085929600/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829902 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829877.256953:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829877.256956:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829877.256960:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085929600:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829877.256966:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085929728:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829877.256983:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829877.256992:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829877/real 1787829877] req@ffff9e35da6f4780 x1874672085929728/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829902 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829877.257004:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4780 x1874672085929728/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829902 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829877.257013:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829877.257016:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829877.257019:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085929728:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829877.383914:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829877.383914:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829877.383931:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829877.383970:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829878.407927:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829878.407928:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829878.407941:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829878.408044:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829879.431893:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829879.431897:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787829879.431915:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829879.431926:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829880.455898:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829880.455901:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:3.0:1787829880.455904:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829880.455965:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829881.479898:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829881.479906:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:3.0:1787829881.479937:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787829881.479943:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829882.375885:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829882.375897:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829882.375901:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829882.375912:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3317 00000100:00080000:1.0:1787829882.375930:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829882.375941:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787829882.375979:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6d00 x1874672085929856/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829882.376007:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829882.376010:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829882.376012:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829882.376016:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3317 00000100:00080000:1.0:1787829882.376019:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829882.376023:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829882.376034:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829882.376046:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6940 x1874672085929984/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829882.376058:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787829882.376067:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085929856:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787829882.376069:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829882.376073:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829882.376081:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3317 00000100:00080000:1.0:1787829882.376095:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829882.376098:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829882.376105:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829882.376118:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4000 x1874672085930112/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829882.376129:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829882.376132:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829882.376134:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829882.376138:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3317 00000100:00080000:1.0:1787829882.376143:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829882.376146:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829882.376151:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000800:00000100:0.0:1787829882.376156:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829882.376163:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7480 x1874672085930240/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829882.376172:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829882.376174:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829882.376176:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829882.376179:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3317 00000100:00080000:1.0:1787829882.376181:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829882.376183:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829882.376185:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:1.0:1787829882.376194:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6580 x1874672085930368/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829882.376201:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829882.376204:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:0.0:1787829882.376205:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829882/real 1787829882] req@ffff9e35da6f6d00 x1874672085929856/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829907 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829882.376207:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829882.376214:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3317 00000100:00080000:1.0:1787829882.376218:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829882.376224:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829882.376228:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829882.376238:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4780 x1874672085930496/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829882.376244:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787829882.376246:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6d00 x1874672085929856/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829907 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829882.376246:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829882.376250:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829882.376256:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3317 00000100:00080000:1.0:1787829882.376259:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829882.376265:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829882.376271:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1787829882.376273:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1787829882.376279:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4f00 x1874672085930624/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829882.376284:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:1.0:1787829882.376291:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787829882.376296:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085929856:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787829882.376296:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829882.376299:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829882.376304:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3317 00000100:00080000:1.0:1787829882.376305:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829882.376308:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829882.376315:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:1.0:1787829882.376323:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f61c0 x1874672085930752/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829882.376328:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829882.376330:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829882.376331:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829882.376334:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3317 00000100:00080000:1.0:1787829882.376336:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829882.376339:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829882.376342:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:0.0:1787829882.376344:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085929984:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829882.376351:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7840 x1874672085930880/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829882.376380:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829882.376399:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829882/real 1787829882] req@ffff9e35da6f6940 x1874672085929984/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829907 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829882.376415:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6940 x1874672085929984/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829907 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829882.376434:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829882.376438:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829882.376442:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085929984:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829882.376450:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085930112:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829882.376474:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829882.376494:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829882/real 1787829882] req@ffff9e35da6f4000 x1874672085930112/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829907 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829882.376507:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4000 x1874672085930112/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829907 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829882.376517:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829882.376520:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829882.376524:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085930112:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829882.376540:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085930240:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829882.376562:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829882.376575:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829882/real 1787829882] req@ffff9e35da6f7480 x1874672085930240/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829907 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829882.376588:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7480 x1874672085930240/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829907 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829882.376607:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829882.376611:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829882.376614:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085930240:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829882.376620:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085930368:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829882.376646:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829882.376657:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829882/real 1787829882] req@ffff9e35da6f6580 x1874672085930368/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829907 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829882.376670:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6580 x1874672085930368/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829907 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829882.376679:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829882.376682:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829882.376685:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085930368:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829882.376691:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085930496:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829882.376712:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829882.376724:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829882/real 1787829882] req@ffff9e35da6f4780 x1874672085930496/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829907 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829882.376737:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4780 x1874672085930496/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829907 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829882.376747:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829882.376751:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829882.376754:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085930496:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829882.376761:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085930624:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829882.376780:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829882.376792:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829882/real 1787829882] req@ffff9e35da6f4f00 x1874672085930624/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829907 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829882.376836:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4f00 x1874672085930624/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829907 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829882.376845:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829882.376848:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829882.376852:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085930624:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829882.376858:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085930752:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829882.376888:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829882.376900:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829882/real 1787829882] req@ffff9e35da6f61c0 x1874672085930752/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829907 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829882.376911:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f61c0 x1874672085930752/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829907 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829882.376919:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829882.376922:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829882.376926:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085930752:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829882.376962:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085930880:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829882.376984:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829882.376998:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829882/real 1787829882] req@ffff9e35da6f7840 x1874672085930880/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829907 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829882.377010:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7840 x1874672085930880/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829907 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829882.377019:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829882.377023:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829882.377025:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085930880:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829882.503910:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829882.503911:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:3.0:1787829882.503911:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787829882.504017:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829883.527917:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829883.527962:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829883.528011:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829883.528019:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829884.551895:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829884.551898:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829884.551941:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787829884.551977:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829885.575910:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:1.0:1787829885.575913:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829885.576025:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829885.576091:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829886.599904:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829886.599938:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829886.599952:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829886.599962:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00080000:2.0:1787829887.495926:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829887.495934:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829887.495938:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829887.495946:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3322 00000100:00080000:2.0:1787829887.495956:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829887.495962:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787829887.495996:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cf00 x1874672085931008/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829887.496027:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829887.496031:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829887.496033:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829887.496038:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3322 00000100:00080000:2.0:1787829887.496042:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829887.496046:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829887.496070:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829887.496084:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e1c0 x1874672085931136/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829887.496097:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085931008:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1787829887.496100:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829887.496108:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829887.496110:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829887.496114:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3322 00000100:00080000:2.0:1787829887.496117:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829887.496120:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829887.496124:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829887.496134:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f480 x1874672085931264/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829887.496141:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829887.496144:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829887.496145:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829887.496148:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3322 00000100:00080000:2.0:1787829887.496150:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829887.496153:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829887.496156:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:2.0:1787829887.496162:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c780 x1874672085931392/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829887.496168:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829887.496171:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829887.496173:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1787829887.496179:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1787829887.496182:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3322 00000100:00080000:2.0:1787829887.496187:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829887.496192:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829887.496198:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:2.0:1787829887.496212:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c3c0 x1874672085931520/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829887.496222:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787829887.496227:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829887/real 1787829887] req@ffff9e35c498cf00 x1874672085931008/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829912 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829887.496228:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829887.496233:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829887.496243:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3322 00000100:00080000:2.0:1787829887.496250:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829887.496257:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787829887.496258:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498cf00 x1874672085931008/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829912 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1787829887.496265:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1787829887.496279:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:2.0:1787829887.496283:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f840 x1874672085931648/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829887.496287:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829887.496297:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085931008:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1787829887.496298:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829887.496307:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829887.496310:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829887.496315:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3322 00000100:00080000:2.0:1787829887.496319:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829887.496323:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829887.496329:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829887.496330:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085931136:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829887.496350:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f0c0 x1874672085931776/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829887.496362:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1787829887.496363:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1787829887.496368:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829887.496371:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829887.496375:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3322 00000100:00080000:2.0:1787829887.496378:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787829887.496384:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829887/real 1787829887] req@ffff9e35c498e1c0 x1874672085931136/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829912 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829887.496384:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829887.496393:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:0.0:1787829887.496400:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e1c0 x1874672085931136/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829912 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829887.496406:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cf00 x1874672085931904/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829887.496410:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829887.496417:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:2.0:1787829887.496419:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787829887.496423:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085931136:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1787829887.496424:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829887.496428:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829887.496434:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3322 00000100:00100000:0.0:1787829887.496435:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085931264:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1787829887.496437:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829887.496442:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829887.496445:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:2.0:1787829887.496455:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e580 x1874672085932032/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829887.496460:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829887.496474:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829887/real 1787829887] req@ffff9e35c498f480 x1874672085931264/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829912 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829887.496483:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f480 x1874672085931264/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829912 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829887.496490:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829887.496492:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829887.496495:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085931264:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829887.496498:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085931392:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829887.496509:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829887.496515:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829887/real 1787829887] req@ffff9e35c498c780 x1874672085931392/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829912 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829887.496523:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c780 x1874672085931392/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829912 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829887.496528:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829887.496529:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829887.496531:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085931392:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829887.496535:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085931520:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829887.496544:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829887.496550:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829887/real 1787829887] req@ffff9e35c498c3c0 x1874672085931520/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829912 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829887.496557:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c3c0 x1874672085931520/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829912 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829887.496562:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829887.496564:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829887.496566:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085931520:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829887.496569:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085931648:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829887.496578:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829887.496584:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829887/real 1787829887] req@ffff9e35c498f840 x1874672085931648/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829912 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829887.496596:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f840 x1874672085931648/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829912 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829887.496602:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829887.496603:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829887.496605:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085931648:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829887.496621:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085931776:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829887.496632:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829887.496638:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829887/real 1787829887] req@ffff9e35c498f0c0 x1874672085931776/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829912 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829887.496646:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f0c0 x1874672085931776/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829912 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829887.496651:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829887.496652:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829887.496654:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085931776:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829887.496658:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085931904:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829887.496668:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829887.496673:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829887/real 1787829887] req@ffff9e35c498cf00 x1874672085931904/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829912 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829887.496680:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498cf00 x1874672085931904/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829912 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829887.496690:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829887.496692:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829887.496694:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085931904:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829887.496697:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085932032:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829887.496706:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829887.496711:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829887/real 1787829887] req@ffff9e35c498e580 x1874672085932032/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829912 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829887.496718:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e580 x1874672085932032/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829912 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829887.496723:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829887.496725:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829887.496727:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085932032:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829887.623888:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829887.623908:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829887.623928:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787829887.623942:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829888.647882:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829888.647885:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829888.647918:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787829888.647918:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829889.671896:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829889.671899:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829889.671923:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829889.671936:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829890.695886:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829890.695886:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829890.695889:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829890.695922:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829891.719881:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829891.719883:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829891.719907:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:1.0:1787829891.719968:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829892.615962:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829892.615973:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829892.615979:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829892.615992:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3327 00000100:00080000:1.0:1787829892.616008:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829892.616030:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787829892.616080:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f43c0 x1874672085932160/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829892.616113:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829892.616117:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829892.616120:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829892.616126:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3327 00000100:00080000:1.0:1787829892.616133:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829892.616138:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829892.616151:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829892.616170:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085932160:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1787829892.616179:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5a40 x1874672085932288/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829892.616196:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829892.616200:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829892.616203:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829892.616209:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3327 00000100:00080000:1.0:1787829892.616216:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829892.616224:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829892.616232:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1787829892.616249:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829892.616255:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5e00 x1874672085932416/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829892.616277:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829892.616281:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829892.616284:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829892.616294:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3327 00000100:00100000:0.0:1787829892.616300:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829892/real 1787829892] req@ffff9e35da6f43c0 x1874672085932160/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829917 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829892.616301:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829892.616325:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787829892.616330:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f43c0 x1874672085932160/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829917 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1787829892.616340:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00080000:0.0:1787829892.616341:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829892.616349:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829892.616354:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085932160:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1787829892.616368:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4b40 x1874672085932544/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829892.616381:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085932288:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787829892.616384:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829892.616392:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829892.616394:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829892.616401:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3327 00000800:00000100:0.0:1787829892.616406:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787829892.616410:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787829892.616422:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829892/real 1787829892] req@ffff9e35da6f5a40 x1874672085932288/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829917 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829892.616422:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829892.616441:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:0.0:1787829892.616452:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5a40 x1874672085932288/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829917 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829892.616462:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1787829892.616463:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f70c0 x1874672085932672/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829892.616468:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829892.616478:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085932288:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829892.616496:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085932416:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787829892.616505:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829892.616511:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829892.616514:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829892.616524:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3327 00000800:00000100:0.0:1787829892.616527:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787829892.616535:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829892.616544:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829892.616551:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829892.616567:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829892/real 1787829892] req@ffff9e35da6f5e00 x1874672085932416/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829917 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829892.616574:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f43c0 x1874672085932800/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829892.616601:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5e00 x1874672085932416/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829917 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829892.616607:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829892.616622:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829892.616628:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:1.0:1787829892.616631:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:0.0:1787829892.616635:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085932416:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787829892.616639:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829892.616653:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3327 00000100:00080000:1.0:1787829892.616662:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829892.616668:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829892.616679:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829892.616681:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085932544:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829892.616708:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7840 x1874672085932928/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829892.616722:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787829892.616728:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829892.616734:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:0.0:1787829892.616740:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829892/real 1787829892] req@ffff9e35da6f4b40 x1874672085932544/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829917 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829892.616750:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787829892.616758:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4b40 x1874672085932544/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829917 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829892.616763:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3327 00000100:00080000:0.0:1787829892.616780:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829892.616780:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829892.616791:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829892.616826:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085932544:192.168.203.141@tcp:8:kworker.0 00000100:00080000:1.0:1787829892.616839:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787829892.616840:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085932672:192.168.203.141@tcp:8:kworker.0 00000008:00100000:1.0:1787829892.616859:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000800:00000100:0.0:1787829892.616875:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829892.616881:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f61c0 x1874672085933056/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829892.616888:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829892/real 1787829892] req@ffff9e35da6f70c0 x1874672085932672/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829917 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829892.616936:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f70c0 x1874672085932672/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829917 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829892.616971:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829892.616972:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829892.616979:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:1.0:1787829892.616986:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:0.0:1787829892.616988:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085932672:192.168.203.141@tcp:8:kworker.0 00000100:00080000:1.0:1787829892.616992:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787829892.616997:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085932800:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787829892.617006:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3327 00000800:00000100:0.0:1787829892.617032:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787829892.617034:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829892.617044:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787829892.617047:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829892/real 1787829892] req@ffff9e35da6f43c0 x1874672085932800/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829917 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1787829892.617059:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:0.0:1787829892.617080:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f43c0 x1874672085932800/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829917 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829892.617087:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1787829892.617090:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4f00 x1874672085933184/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829892.617092:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829892.617102:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085932800:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829892.617149:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085932928:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829892.617170:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829892.617179:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829892/real 1787829892] req@ffff9e35da6f7840 x1874672085932928/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829917 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829892.617187:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7840 x1874672085932928/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829917 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829892.617192:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829892.617194:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829892.617196:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085932928:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829892.617200:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085933056:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829892.617211:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829892.617217:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829892/real 1787829892] req@ffff9e35da6f61c0 x1874672085933056/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829917 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829892.617224:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f61c0 x1874672085933056/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829917 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829892.617229:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829892.617231:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829892.617233:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085933056:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829892.617237:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085933184:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829892.617247:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829892.617252:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829892/real 1787829892] req@ffff9e35da6f4f00 x1874672085933184/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829917 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829892.617259:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4f00 x1874672085933184/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829917 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829892.617265:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829892.617267:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829892.617268:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085933184:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829892.743886:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829892.743931:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829892.743949:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829892.744072:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829893.767894:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829893.767905:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829893.767950:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829893.767986:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829894.791909:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829894.791909:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829894.791916:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787829894.791973:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829895.815912:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829895.815920:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829895.815938:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829895.815943:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829896.839889:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829896.839893:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829896.839898:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829896.839906:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00080000:3.0:1787829897.735949:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829897.735961:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787829897.735967:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829897.735989:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3333 00000100:00080000:3.0:1787829897.736004:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829897.736014:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787829897.736063:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499a40 x1874672085933312/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829897.736095:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829897.736098:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829897.736100:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829897.736103:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3333 00000100:00080000:3.0:1787829897.736107:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829897.736110:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829897.736121:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829897.736133:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498f00 x1874672085933440/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829897.736141:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085933312:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1787829897.736145:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829897.736149:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787829897.736152:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829897.736156:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3333 00000100:00080000:3.0:1787829897.736158:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829897.736160:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829897.736164:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829897.736173:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498780 x1874672085933568/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829897.736181:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829897.736184:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787829897.736185:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829897.736189:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3333 00000100:00080000:3.0:1787829897.736191:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829897.736194:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829897.736196:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:3.0:1787829897.736204:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b480 x1874672085933696/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829897.736210:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829897.736215:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000800:00000100:0.0:1787829897.736216:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1787829897.736218:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829897.736224:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3333 00000100:00080000:3.0:1787829897.736226:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829897.736229:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829897.736232:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:3.0:1787829897.736242:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c64992c0 x1874672085933824/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829897.736248:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829897.736251:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829897.736252:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829897.736255:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3333 00000100:00080000:3.0:1787829897.736257:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829897.736260:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829897.736263:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829897.736275:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829897/real 1787829897] req@ffff9e35c6499a40 x1874672085933312/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829922 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829897.736275:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c64983c0 x1874672085933952/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829897.736301:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829897.736305:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787829897.736307:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829897.736310:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3333 00000100:00080000:3.0:1787829897.736312:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787829897.736317:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499a40 x1874672085933312/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829922 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829897.736317:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829897.736324:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829897.736335:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649ad00 x1874672085934080/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829897.736336:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829897.736344:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:3.0:1787829897.736349:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787829897.736352:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085933312:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1787829897.736353:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787829897.736358:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829897.736363:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3333 00000100:00080000:3.0:1787829897.736366:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829897.736368:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829897.736371:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:3.0:1787829897.736378:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498000 x1874672085934208/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829897.736382:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085933440:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1787829897.736393:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829897.736395:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787829897.736397:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829897.736400:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3333 00000100:00080000:3.0:1787829897.736403:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829897.736406:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829897.736409:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000800:00000100:0.0:1787829897.736413:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787829897.736420:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499a40 x1874672085934336/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829897.736428:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829897/real 1787829897] req@ffff9e35c6498f00 x1874672085933440/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829922 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829897.736439:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498f00 x1874672085933440/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829922 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829897.736446:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829897.736449:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829897.736451:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085933440:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829897.736456:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085933568:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829897.736469:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829897.736476:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829897/real 1787829897] req@ffff9e35c6498780 x1874672085933568/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829922 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829897.736483:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498780 x1874672085933568/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829922 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829897.736493:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829897.736496:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829897.736498:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085933568:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829897.736501:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085933696:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829897.736513:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829897.736519:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829897/real 1787829897] req@ffff9e35c649b480 x1874672085933696/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829922 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829897.736526:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b480 x1874672085933696/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829922 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829897.736532:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829897.736534:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829897.736536:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085933696:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829897.736539:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c64992c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085933824:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829897.736549:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829897.736554:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829897/real 1787829897] req@ffff9e35c64992c0 x1874672085933824/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829922 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829897.736561:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c64992c0 x1874672085933824/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829922 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829897.736567:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829897.736569:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829897.736571:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c64992c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085933824:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829897.736575:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c64983c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085933952:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829897.736596:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829897.736602:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829897/real 1787829897] req@ffff9e35c64983c0 x1874672085933952/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829922 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829897.736610:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c64983c0 x1874672085933952/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829922 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829897.736615:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829897.736617:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829897.736619:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c64983c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085933952:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829897.736622:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085934080:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829897.736633:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829897.736638:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829897/real 1787829897] req@ffff9e35c649ad00 x1874672085934080/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829922 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829897.736645:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649ad00 x1874672085934080/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829922 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829897.736650:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829897.736652:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829897.736654:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085934080:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829897.736670:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085934208:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829897.736680:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829897.736686:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829897/real 1787829897] req@ffff9e35c6498000 x1874672085934208/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829922 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829897.736694:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498000 x1874672085934208/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829922 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829897.736699:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829897.736701:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829897.736703:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085934208:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829897.736706:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085934336:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829897.736715:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829897.736721:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829897/real 1787829897] req@ffff9e35c6499a40 x1874672085934336/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829922 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829897.736728:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499a40 x1874672085934336/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829922 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829897.736733:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829897.736735:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829897.736737:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085934336:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829897.863921:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829897.863966:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787829897.864018:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829897.864060:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829898.887921:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829898.887945:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829898.887953:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787829898.888009:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829899.911890:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829899.911894:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829899.911926:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829899.911933:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829900.935881:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829900.935886:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829900.935889:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787829900.935962:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829901.959886:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829901.959886:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:3.0:1787829901.959886:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787829901.959989:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829902.855935:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829902.855962:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829902.855965:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829902.855973:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3338 00000100:00080000:1.0:1787829902.855983:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829902.855988:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787829902.856017:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4780 x1874672085934464/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829902.856035:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829902.856038:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829902.856039:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829902.856042:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3338 00000100:00080000:1.0:1787829902.856044:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829902.856046:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829902.856054:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829902.856062:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6580 x1874672085934592/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829902.856070:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085934464:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787829902.856071:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829902.856076:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829902.856079:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829902.856083:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3338 00000100:00080000:1.0:1787829902.856085:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829902.856086:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829902.856089:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829902.856097:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7480 x1874672085934720/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829902.856103:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829902.856105:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829902.856106:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829902.856108:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3338 00000100:00080000:1.0:1787829902.856123:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829902.856125:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829902.856128:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000800:00000100:0.0:1787829902.856133:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829902.856137:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4000 x1874672085934848/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829902.856144:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829902.856147:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829902.856148:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829902.856151:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3338 00000100:00080000:1.0:1787829902.856153:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829902.856154:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829902.856157:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:1.0:1787829902.856164:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6940 x1874672085934976/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829902.856170:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787829902.856175:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829902/real 1787829902] req@ffff9e35da6f4780 x1874672085934464/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829927 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829902.856175:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829902.856176:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829902.856183:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3338 00000100:00080000:1.0:1787829902.856186:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829902.856191:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829902.856195:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829902.856205:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4780 x1874672085934464/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829927 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829902.856205:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6d00 x1874672085935104/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829902.856234:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829902.856234:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829902.856240:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829902.856242:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829902.856243:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:1.0:1787829902.856246:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3338 00000100:00080000:1.0:1787829902.856249:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787829902.856251:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085934464:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787829902.856253:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829902.856260:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829902.856267:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4f00 x1874672085935232/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829902.856272:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829902.856274:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829902.856275:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829902.856280:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3338 00000100:00100000:0.0:1787829902.856281:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085934592:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787829902.856282:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829902.856287:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829902.856291:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:1.0:1787829902.856298:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f61c0 x1874672085935360/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829902.856303:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829902.856305:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829902.856307:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829902.856310:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3338 00000100:00080000:1.0:1787829902.856313:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829902.856318:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:0.0:1787829902.856320:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:1.0:1787829902.856322:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:1.0:1787829902.856331:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7840 x1874672085935488/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829902.856337:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829902/real 1787829902] req@ffff9e35da6f6580 x1874672085934592/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829927 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829902.856359:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6580 x1874672085934592/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829927 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829902.856368:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829902.856371:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829902.856374:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085934592:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829902.856380:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085934720:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829902.856396:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829902.856405:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829902/real 1787829902] req@ffff9e35da6f7480 x1874672085934720/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829927 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829902.856415:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7480 x1874672085934720/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829927 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829902.856429:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829902.856433:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829902.856435:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085934720:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829902.856440:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085934848:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829902.856454:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829902.856462:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829902/real 1787829902] req@ffff9e35da6f4000 x1874672085934848/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829927 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829902.856473:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4000 x1874672085934848/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829927 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829902.856481:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829902.856484:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829902.856487:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085934848:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829902.856492:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085934976:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829902.856512:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829902.856520:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829902/real 1787829902] req@ffff9e35da6f6940 x1874672085934976/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829927 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829902.856530:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6940 x1874672085934976/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829927 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829902.856538:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829902.856541:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829902.856544:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085934976:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829902.856549:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085935104:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829902.856563:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829902.856572:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829902/real 1787829902] req@ffff9e35da6f6d00 x1874672085935104/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829927 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829902.856583:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6d00 x1874672085935104/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829927 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829902.856591:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829902.856594:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829902.856597:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085935104:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829902.856602:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085935232:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829902.856617:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829902.856625:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829902/real 1787829902] req@ffff9e35da6f4f00 x1874672085935232/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829927 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829902.856636:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4f00 x1874672085935232/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829927 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829902.856644:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829902.856646:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829902.856649:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085935232:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829902.856671:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085935360:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829902.856685:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829902.856694:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829902/real 1787829902] req@ffff9e35da6f61c0 x1874672085935360/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829927 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829902.856705:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f61c0 x1874672085935360/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829927 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829902.856713:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829902.856715:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829902.856718:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085935360:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829902.856723:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085935488:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829902.856736:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829902.856744:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829902/real 1787829902] req@ffff9e35da6f7840 x1874672085935488/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829927 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829902.856754:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7840 x1874672085935488/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829927 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829902.856762:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829902.856765:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829902.856768:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085935488:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829902.983858:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829902.983862:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:3.0:1787829902.983889:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829902.983960:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829904.007929:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829904.007944:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:3.0:1787829904.007971:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787829904.008076:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829905.031890:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829905.031963:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829905.031985:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829905.032062:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829906.055881:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829906.055906:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829906.055916:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829906.055934:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829907.079855:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829907.079860:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:3.0:1787829907.079872:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787829907.079961:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829907.975949:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829907.975958:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787829907.975961:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829907.975969:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3343 00000100:00080000:0.0:1787829907.975978:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829907.975983:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787829907.976049:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a1c0 x1874672085935616/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829907.976090:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829907.976094:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829907.976096:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829907.976100:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3343 00000100:00080000:0.0:1787829907.976104:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829907.976108:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829907.976118:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829907.976123:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085935616:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1787829907.976131:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7192c0 x1874672085935744/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829907.976140:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829907.976143:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787829907.976145:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829907.976148:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3343 00000100:00080000:0.0:1787829907.976151:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829907.976154:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829907.976157:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829907.976165:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718f00 x1874672085935872/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787829907.976177:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787829907.976181:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829907.976184:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829907.976185:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829907.976189:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3343 00000100:00080000:0.0:1787829907.976191:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829907.976194:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829907.976199:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:0.0:1787829907.976208:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a940 x1874672085936000/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829907.976214:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829907/real 1787829907] req@ffff9e35da71a1c0 x1874672085935616/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829932 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829907.976216:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829907.976220:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829907.976222:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829907.976225:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3343 00000100:00080000:0.0:1787829907.976228:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829907.976230:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787829907.976231:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a1c0 x1874672085935616/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829932 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1787829907.976234:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00080000:1.0:1787829907.976240:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1787829907.976242:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718780 x1874672085936128/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829907.976243:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829907.976247:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085935616:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1787829907.976251:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829907.976254:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829907.976255:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829907.976258:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3343 00000100:00080000:0.0:1787829907.976261:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829907.976264:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787829907.976266:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085935744:192.168.203.141@tcp:38:kworker.0 00000008:00100000:0.0:1787829907.976268:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829907.976276:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da719a40 x1874672085936256/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829907.976285:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1787829907.976285:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787829907.976288:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787829907.976290:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829907.976293:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3343 00000100:00080000:0.0:1787829907.976296:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1787829907.976296:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829907/real 1787829907] req@ffff9e35da7192c0 x1874672085935744/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829932 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829907.976299:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829907.976303:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829907.976311:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7183c0 x1874672085936384/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829907.976314:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7192c0 x1874672085935744/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829932 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829907.976320:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829907.976322:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829907.976323:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829907.976325:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829907.976325:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829907.976328:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085935744:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787829907.976329:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3343 00000100:00080000:0.0:1787829907.976331:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1787829907.976333:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085935872:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787829907.976335:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829907.976339:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:0.0:1787829907.976347:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718000 x1874672085936512/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787829907.976349:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829907.976358:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829907/real 1787829907] req@ffff9e35da718f00 x1874672085935872/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829932 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829907.976362:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829907.976366:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829907.976368:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787829907.976369:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672085935872/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829932 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829907.976371:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3343 00000100:00080000:0.0:1787829907.976374:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829907.976377:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829907.976381:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00080000:1.0:1787829907.976386:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829907.976392:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829907.976396:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085935872:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829907.976401:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085936000:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829907.976421:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829907.976424:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a580 x1874672085936640/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829907.976430:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829907/real 1787829907] req@ffff9e35da71a940 x1874672085936000/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829932 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829907.976440:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672085936000/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829932 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829907.976452:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829907.976455:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829907.976458:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085936000:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829907.976463:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085936128:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829907.976480:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829907.976488:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829907/real 1787829907] req@ffff9e35da718780 x1874672085936128/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829932 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829907.976495:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718780 x1874672085936128/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829932 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829907.976500:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829907.976502:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829907.976504:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085936128:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829907.976530:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085936256:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787829907.976550:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829907.976560:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829907/real 1787829907] req@ffff9e35da719a40 x1874672085936256/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829932 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829907.976571:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672085936256/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829932 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829907.976578:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829907.976581:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829907.976584:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085936256:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829907.976590:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085936384:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787829907.976603:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829907.976610:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829907/real 1787829907] req@ffff9e35da7183c0 x1874672085936384/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829932 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829907.976616:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672085936384/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829932 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829907.976621:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829907.976622:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829907.976624:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085936384:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829907.976627:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085936512:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829907.976635:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829907.976640:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829907/real 1787829907] req@ffff9e35da718000 x1874672085936512/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829932 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829907.976646:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718000 x1874672085936512/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829932 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829907.976650:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829907.976652:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829907.976654:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085936512:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829907.976657:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085936640:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829907.976670:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829907.976677:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829907/real 1787829907] req@ffff9e35da71a580 x1874672085936640/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829932 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829907.976684:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672085936640/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829932 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829907.976688:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829907.976690:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829907.976691:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085936640:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829908.103899:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829908.103948:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829908.103950:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829908.103954:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829909.127929:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829909.127929:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829909.127960:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829909.127963:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829910.151887:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829910.151913:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829910.151916:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829910.151942:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829911.175912:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829911.175919:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829911.175963:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787829911.175963:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829912.199905:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829912.199948:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829912.199949:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829912.200015:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787829913.095939:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829913.095948:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829913.095952:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829913.095962:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3348 00000100:00080000:1.0:1787829913.095973:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829913.095979:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787829913.096014:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f43c0 x1874672085936768/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829913.096043:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829913.096047:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829913.096049:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829913.096052:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3348 00000100:00080000:1.0:1787829913.096056:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829913.096059:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829913.096068:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829913.096079:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f70c0 x1874672085936896/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829913.096088:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829913.096091:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829913.096093:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829913.096097:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3348 00000100:00100000:0.0:1787829913.096102:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085936768:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787829913.096103:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829913.096109:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829913.096115:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829913.096125:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4b40 x1874672085937024/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829913.096133:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829913.096136:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829913.096138:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829913.096141:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3348 00000100:00080000:1.0:1787829913.096160:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829913.096164:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829913.096169:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000800:00000100:0.0:1787829913.096172:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829913.096184:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5e00 x1874672085937152/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829913.096206:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829913.096214:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:0.0:1787829913.096216:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829913/real 1787829913] req@ffff9e35da6f43c0 x1874672085936768/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829938 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829913.096217:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829913.096224:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3348 00000100:00080000:1.0:1787829913.096231:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829913.096237:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829913.096242:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:0.0:1787829913.096245:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f43c0 x1874672085936768/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829938 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829913.096255:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5a40 x1874672085937280/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829913.096257:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829913.096265:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829913.096273:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085936768:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787829913.096274:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829913.096281:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:0.0:1787829913.096284:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085936896:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787829913.096285:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829913.096293:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3348 00000100:00080000:1.0:1787829913.096297:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829913.096301:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829913.096306:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1787829913.096308:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829913.096321:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829913/real 1787829913] req@ffff9e35da6f70c0 x1874672085936896/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829938 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829913.096324:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7840 x1874672085937408/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829913.096338:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f70c0 x1874672085936896/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829938 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829913.096340:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829913.096349:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787829913.096352:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829913.096352:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829913.096360:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3348 00000100:00080000:0.0:1787829913.096362:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:1.0:1787829913.096367:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787829913.096368:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085936896:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787829913.096375:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829913.096381:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829913.096392:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f61c0 x1874672085937536/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829913.096394:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085937024:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787829913.096409:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829913.096413:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829913.096415:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829913.096419:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3348 00000100:00080000:1.0:1787829913.096424:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829913.096427:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829913.096432:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000800:00000100:0.0:1787829913.096437:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829913.096447:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4f00 x1874672085937664/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829913.096451:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829913/real 1787829913] req@ffff9e35da6f4b40 x1874672085937024/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829938 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829913.096470:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787829913.096474:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4b40 x1874672085937024/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829938 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829913.096477:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829913.096482:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829913.096486:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829913.096490:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3348 00000100:00080000:0.0:1787829913.096493:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:1.0:1787829913.096498:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787829913.096499:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085937024:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787829913.096506:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787829913.096511:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085937152:192.168.203.141@tcp:8:kworker.0 00000008:00100000:1.0:1787829913.096512:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:1.0:1787829913.096529:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6d00 x1874672085937792/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829913.096541:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829913.096563:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829913/real 1787829913] req@ffff9e35da6f5e00 x1874672085937152/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829938 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829913.096577:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5e00 x1874672085937152/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829938 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829913.096586:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829913.096589:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829913.096592:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085937152:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829913.096599:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085937280:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829913.096625:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829913.096636:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829913/real 1787829913] req@ffff9e35da6f5a40 x1874672085937280/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829938 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829913.096647:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5a40 x1874672085937280/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829938 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829913.096663:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829913.096665:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829913.096667:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085937280:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829913.096671:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085937408:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829913.096684:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829913.096694:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829913/real 1787829913] req@ffff9e35da6f7840 x1874672085937408/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829938 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829913.096704:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7840 x1874672085937408/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829938 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829913.096718:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829913.096721:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829913.096724:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085937408:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829913.096744:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085937536:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829913.096757:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829913.096764:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829913/real 1787829913] req@ffff9e35da6f61c0 x1874672085937536/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829938 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829913.096771:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f61c0 x1874672085937536/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829938 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829913.096776:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829913.096778:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829913.096780:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085937536:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829913.096782:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085937664:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829913.096791:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829913.096795:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829913/real 1787829913] req@ffff9e35da6f4f00 x1874672085937664/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829938 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829913.096818:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4f00 x1874672085937664/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829938 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829913.096823:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829913.096825:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829913.096827:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085937664:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829913.096831:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085937792:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829913.096841:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829913.096847:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829913/real 1787829913] req@ffff9e35da6f6d00 x1874672085937792/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829938 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829913.096853:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6d00 x1874672085937792/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829938 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829913.096858:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829913.096860:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829913.096862:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085937792:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829913.223926:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829913.223926:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829913.223934:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829913.223981:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829914.247904:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829914.247936:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829914.247974:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787829914.248207:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829915.271929:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829915.271929:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829915.271934:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829915.271959:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829916.295898:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829916.295898:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829916.295911:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787829916.295942:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829917.319904:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829917.319910:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829917.319922:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787829917.319944:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829918.215947:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829918.215954:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829918.215957:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829918.215964:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3353 00000100:00080000:1.0:1787829918.215972:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829918.215977:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787829918.216006:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6940 x1874672085937920/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829918.216027:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829918.216029:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829918.216031:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829918.216033:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3353 00000100:00080000:1.0:1787829918.216035:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829918.216037:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829918.216044:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829918.216051:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4000 x1874672085938048/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829918.216058:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829918.216060:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829918.216061:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829918.216064:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3353 00000100:00080000:1.0:1787829918.216066:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829918.216067:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829918.216070:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829918.216076:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7480 x1874672085938176/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829918.216083:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787829918.216084:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085937920:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787829918.216088:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829918.216090:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829918.216095:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3353 00000100:00080000:1.0:1787829918.216097:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829918.216099:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829918.216101:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:1.0:1787829918.216108:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6580 x1874672085938304/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829918.216113:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829918.216115:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829918.216116:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829918.216119:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3353 00000100:00080000:1.0:1787829918.216121:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829918.216122:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829918.216125:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:1.0:1787829918.216132:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4780 x1874672085938432/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829918.216138:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829918.216139:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829918.216141:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829918.216143:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3353 00000100:00080000:1.0:1787829918.216145:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829918.216147:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829918.216149:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829918.216155:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6d00 x1874672085938560/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829918.216160:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829918.216162:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829918.216163:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829918.216165:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3353 00000100:00080000:1.0:1787829918.216167:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829918.216169:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829918.216171:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829918.216177:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4f00 x1874672085938688/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829918.216183:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829918.216185:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829918.216186:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829918.216190:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3353 00000100:00100000:0.0:1787829918.216191:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085938048:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787829918.216193:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829918.216197:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829918.216202:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:1.0:1787829918.216210:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f61c0 x1874672085938816/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829918.216215:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829918.216218:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829918.216219:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829918.216221:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3353 00000100:00080000:1.0:1787829918.216224:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829918.216226:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829918.216229:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:1.0:1787829918.216236:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7840 x1874672085938944/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829918.216240:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085938176:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829918.216269:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085938304:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829918.216293:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085938432:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829918.216314:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085938560:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829918.216332:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085938688:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829918.216345:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085938816:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829918.216366:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085938944:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829918.343887:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829918.343887:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829918.343887:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829918.343938:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829919.367920:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829919.367921:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829919.367921:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787829919.368003:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829920.391896:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829920.391921:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829920.391923:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787829920.391933:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1787829921.288039:0:2398:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.141:988 00000400:00000100:1.0:1787829921.288074:0:2398:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.141@tcp at host 192.168.203.141 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1787829921.288088:0:2398:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:1.0:1787829921.288130:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000800:00000100:1.0:1787829921.288147:0:2398:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:1.0:1787829921.288160:0:2398:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:1.0:1787829921.288168:0:2398:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:0.0:1787829921.288171:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829918/real 1787829921] req@ffff9e35da6f7840 x1874672085938944/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829943 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:1.0:1787829921.288179:0:2398:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:1.0:1787829921.288190:0:2398:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:0.0:1787829921.288204:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7840 x1874672085938944/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829943 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:1.0:1787829921.288206:0:2398:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:1.0:1787829921.288218:0:2398:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00080000:0.0:1787829921.288222:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829921.288229:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829921.288234:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085938944:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829921.288243:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829918/real 1787829921] req@ffff9e35da6f7480 x1874672085938176/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829943 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829921.288257:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7480 x1874672085938176/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829943 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829921.288277:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829921.288281:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829921.288285:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085938176:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829921.288293:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829918/real 1787829921] req@ffff9e35da6f6580 x1874672085938304/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829943 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829921.288314:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6580 x1874672085938304/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829943 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829921.288323:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829921.288327:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829921.288331:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085938304:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829921.288337:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829918/real 1787829921] req@ffff9e35da6f4780 x1874672085938432/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829943 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829921.288350:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4780 x1874672085938432/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829943 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829921.288359:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829921.288363:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829921.288367:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085938432:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829921.288375:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829918/real 1787829921] req@ffff9e35da6f6d00 x1874672085938560/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829943 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829921.288385:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6d00 x1874672085938560/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829943 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829921.288392:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829921.288396:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829921.288399:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085938560:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829921.288404:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829918/real 1787829921] req@ffff9e35da6f4f00 x1874672085938688/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829943 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829921.288415:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4f00 x1874672085938688/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829943 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829921.288421:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829921.288424:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829921.288427:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085938688:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829921.288433:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829918/real 1787829921] req@ffff9e35da6f61c0 x1874672085938816/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829943 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829921.288443:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f61c0 x1874672085938816/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829943 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829921.288450:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829921.288453:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829921.288456:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085938816:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829921.288462:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829918/real 1787829921] req@ffff9e35da6f6940 x1874672085937920/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829943 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829921.288473:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6940 x1874672085937920/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829943 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829921.288481:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829921.288484:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829921.288487:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085937920:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1787829921.288493:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829918/real 1787829921] req@ffff9e35da6f4000 x1874672085938048/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829943 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829921.288505:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4000 x1874672085938048/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829943 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829921.288512:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829921.288515:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829921.288518:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085938048:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829921.415911:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829921.415911:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829921.415911:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829921.415951:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829922.439889:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829922.439890:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829922.439917:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:1.0:1787829922.439934:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829923.335923:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829923.335931:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787829923.335933:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829923.335941:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3358 00000100:00080000:0.0:1787829923.335949:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829923.335954:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787829923.335980:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718b40 x1874672085939072/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829923.336005:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829923.336008:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829923.336011:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829923.336015:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3358 00000100:00080000:0.0:1787829923.336018:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829923.336023:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829923.336034:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829923.336047:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085939072:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1787829923.336054:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a580 x1874672085939200/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829923.336066:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829923.336070:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787829923.336072:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829923.336076:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3358 00000100:00080000:0.0:1787829923.336080:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829923.336084:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829923.336089:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829923.336102:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718000 x1874672085939328/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787829923.336113:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787829923.336114:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829923.336120:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829923.336122:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829923.336127:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3358 00000100:00080000:0.0:1787829923.336132:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829923.336136:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829923.336141:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:0.0:1787829923.336155:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7183c0 x1874672085939456/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829923.336166:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829923/real 1787829923] req@ffff9e35da718b40 x1874672085939072/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829948 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829923.336167:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829923.336176:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829923.336180:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829923.336189:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3358 00000100:00080000:0.0:1787829923.336193:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1787829923.336198:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718b40 x1874672085939072/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829948 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829923.336201:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829923.336209:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00080000:1.0:1787829923.336215:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829923.336221:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829923.336225:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da719a40 x1874672085939584/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829923.336229:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085939072:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1787829923.336244:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829923.336248:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829923.336250:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829923.336256:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3358 00000100:00100000:1.0:1787829923.336258:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085939200:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787829923.336261:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829923.336268:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829923.336274:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1787829923.336287:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829923.336291:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718780 x1874672085939712/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829923.336303:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787829923.336305:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829923/real 1787829923] req@ffff9e35da71a580 x1874672085939200/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829948 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829923.336308:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787829923.336312:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829923.336322:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3358 00000100:00080000:0.0:1787829923.336327:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1787829923.336329:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672085939200/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829948 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829923.336334:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829923.336343:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00080000:1.0:1787829923.336346:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829923.336352:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829923.336358:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a940 x1874672085939840/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829923.336359:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085939200:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829923.336372:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085939328:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787829923.336376:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829923.336381:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829923.336384:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829923.336390:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3358 00000100:00080000:0.0:1787829923.336394:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1787829923.336398:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787829923.336402:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829923.336409:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:1.0:1787829923.336414:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829923/real 1787829923] req@ffff9e35da718000 x1874672085939328/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829948 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829923.336424:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718f00 x1874672085939968/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829923.336431:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718000 x1874672085939328/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829948 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829923.336440:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829923.336445:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829923.336446:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829923.336450:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829923.336453:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:0.0:1787829923.336458:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3358 00000100:00100000:1.0:1787829923.336461:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085939328:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787829923.336464:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829923.336471:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787829923.336474:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085939456:192.168.203.141@tcp:8:kworker.0 00000008:00100000:0.0:1787829923.336485:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000800:00000100:1.0:1787829923.336498:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829923.336501:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7192c0 x1874672085940096/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829923.336514:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829923/real 1787829923] req@ffff9e35da7183c0 x1874672085939456/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829948 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829923.336527:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672085939456/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829948 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829923.336535:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829923.336538:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829923.336542:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085939456:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829923.336549:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085939584:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829923.336568:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829923.336579:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829923/real 1787829923] req@ffff9e35da719a40 x1874672085939584/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829948 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829923.336590:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672085939584/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829948 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829923.336598:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829923.336604:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829923.336606:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085939584:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829923.336622:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085939712:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787829923.336639:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829923.336649:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829923/real 1787829923] req@ffff9e35da718780 x1874672085939712/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829948 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829923.336661:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718780 x1874672085939712/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829948 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829923.336668:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829923.336670:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829923.336673:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085939712:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829923.336678:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085939840:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787829923.336695:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829923.336704:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829923/real 1787829923] req@ffff9e35da71a940 x1874672085939840/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829948 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829923.336713:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672085939840/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829948 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829923.336719:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829923.336722:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829923.336725:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085939840:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829923.336730:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085939968:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829923.336748:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829923.336756:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829923/real 1787829923] req@ffff9e35da718f00 x1874672085939968/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829948 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829923.336766:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672085939968/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829948 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829923.336773:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829923.336776:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829923.336778:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085939968:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829923.336783:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085940096:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829923.336833:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829923.336845:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829923/real 1787829923] req@ffff9e35da7192c0 x1874672085940096/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829948 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829923.336855:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7192c0 x1874672085940096/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829948 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829923.336862:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829923.336865:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829923.336868:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085940096:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829923.463902:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829923.463903:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829923.463932:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787829923.463949:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829924.487869:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829924.487877:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829924.487900:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787829924.487924:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829925.511909:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829925.511933:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829925.511939:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787829925.511963:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829926.535893:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829926.535893:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787829926.535978:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:0.0:1787829926.536048:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829927.559902:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829927.559902:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787829927.559902:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787829927.559993:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829928.455934:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829928.455941:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787829928.455944:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829928.455952:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3363 00000100:00080000:0.0:1787829928.455966:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829928.455972:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787829928.455999:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a1c0 x1874672085940224/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829928.456017:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829928.456020:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829928.456022:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829928.456024:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3363 00000100:00080000:0.0:1787829928.456027:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829928.456029:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829928.456038:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829928.456048:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7192c0 x1874672085940352/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829928.456056:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085940224:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1787829928.456057:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829928.456064:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787829928.456067:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829928.456071:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3363 00000100:00080000:0.0:1787829928.456073:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829928.456076:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829928.456081:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829928.456089:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718f00 x1874672085940480/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829928.456095:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829928.456097:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829928.456099:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829928.456101:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3363 00000100:00080000:0.0:1787829928.456111:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829928.456113:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829928.456116:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000800:00000100:1.0:1787829928.456122:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829928.456124:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a940 x1874672085940608/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829928.456131:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829928.456134:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829928.456135:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829928.456138:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3363 00000100:00080000:0.0:1787829928.456140:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829928.456142:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829928.456145:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:0.0:1787829928.456152:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718780 x1874672085940736/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829928.456158:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829928.456161:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:1.0:1787829928.456164:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829928/real 1787829928] req@ffff9e35da71a1c0 x1874672085940224/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829953 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829928.456168:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829928.456179:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3363 00000100:00080000:0.0:1787829928.456183:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1787829928.456188:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a1c0 x1874672085940224/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829953 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829928.456211:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829928.456215:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829928.456219:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085940224:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1787829928.456224:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829928.456229:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829928.456239:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da719a40 x1874672085940864/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829928.456247:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829928.456251:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:1.0:1787829928.456251:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085940352:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787829928.456254:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829928.456261:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3363 00000100:00080000:0.0:1787829928.456264:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829928.456268:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829928.456271:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829928.456285:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7183c0 x1874672085940992/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787829928.456286:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787829928.456297:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829928.456301:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829928.456303:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787829928.456306:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829928/real 1787829928] req@ffff9e35da7192c0 x1874672085940352/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829953 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829928.456310:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3363 00000100:00080000:0.0:1787829928.456318:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829928.456324:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787829928.456329:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7192c0 x1874672085940352/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829953 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1787829928.456332:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00080000:1.0:1787829928.456340:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829928.456346:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829928.456351:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085940352:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829928.456352:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718000 x1874672085941120/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829928.456362:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787829928.456363:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085940480:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787829928.456366:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829928.456369:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829928.456377:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3363 00000100:00080000:0.0:1787829928.456380:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829928.456383:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829928.456387:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:0.0:1787829928.456396:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a580 x1874672085941248/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787829928.456405:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829928.456424:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829928/real 1787829928] req@ffff9e35da718f00 x1874672085940480/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829953 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829928.456440:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672085940480/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829953 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829928.456449:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829928.456453:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829928.456458:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085940480:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829928.456465:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085940608:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829928.456486:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829928.456500:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829928/real 1787829928] req@ffff9e35da71a940 x1874672085940608/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829953 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829928.456520:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672085940608/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829953 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829928.456528:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829928.456531:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829928.456534:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085940608:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829928.456539:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085940736:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829928.456556:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829928.456565:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829928/real 1787829928] req@ffff9e35da718780 x1874672085940736/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829953 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829928.456582:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718780 x1874672085940736/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829953 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829928.456590:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829928.456592:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829928.456595:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085940736:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829928.456609:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085940864:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787829928.456625:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829928.456636:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829928/real 1787829928] req@ffff9e35da719a40 x1874672085940864/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829953 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829928.456647:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672085940864/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829953 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829928.456654:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829928.456657:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829928.456658:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085940864:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829928.456662:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085940992:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787829928.456676:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829928.456682:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829928/real 1787829928] req@ffff9e35da7183c0 x1874672085940992/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829953 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829928.456688:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672085940992/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829953 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829928.456697:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829928.456698:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829928.456700:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085940992:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829928.456703:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085941120:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829928.456711:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829928.456716:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829928/real 1787829928] req@ffff9e35da718000 x1874672085941120/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829953 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829928.456722:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718000 x1874672085941120/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829953 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829928.456727:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829928.456728:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829928.456730:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085941120:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829928.456733:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085941248:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787829928.456740:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829928.456745:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829928/real 1787829928] req@ffff9e35da71a580 x1874672085941248/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829953 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829928.456751:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672085941248/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829953 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829928.456755:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829928.456757:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829928.456759:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085941248:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829928.583882:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787829928.583941:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829928.583993:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829928.584048:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829929.607937:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829929.607937:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:3.0:1787829929.607954:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787829929.608077:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829930.631957:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829930.631963:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:3.0:1787829930.631963:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787829930.632095:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829931.655939:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:3.0:1787829931.655939:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829931.655945:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:1.0:1787829931.656058:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829932.679915:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829932.679915:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829932.679962:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829932.679966:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829933.575918:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829933.575930:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829933.575935:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829933.575948:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3368 00000100:00080000:1.0:1787829933.575959:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829933.575975:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787829933.576022:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5a40 x1874672085941376/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829933.576052:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829933.576056:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829933.576058:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829933.576063:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3368 00000100:00080000:1.0:1787829933.576068:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829933.576073:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829933.576085:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829933.576095:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085941376:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1787829933.576101:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5e00 x1874672085941504/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829933.576114:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829933.576118:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829933.576120:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829933.576125:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3368 00000100:00080000:1.0:1787829933.576129:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829933.576133:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829933.576138:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829933.576148:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4b40 x1874672085941632/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829933.576158:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829933.576161:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829933.576162:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829933.576166:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3368 00000100:00080000:1.0:1787829933.576169:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829933.576175:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:0.0:1787829933.576176:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:1.0:1787829933.576180:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:1.0:1787829933.576191:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f70c0 x1874672085941760/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829933.576201:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829933.576204:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829933.576206:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829933.576210:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3368 00000100:00080000:1.0:1787829933.576213:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829933.576216:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829933.576220:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:0.0:1787829933.576230:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829933/real 1787829933] req@ffff9e35da6f5a40 x1874672085941376/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829958 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829933.576241:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f43c0 x1874672085941888/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829933.576252:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787829933.576255:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5a40 x1874672085941376/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829958 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829933.576257:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829933.576261:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829933.576267:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3368 00000100:00080000:1.0:1787829933.576274:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829933.576280:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:0.0:1787829933.576281:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:1.0:1787829933.576286:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1787829933.576290:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829933.576297:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085941376:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1787829933.576304:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4000 x1874672085942016/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829933.576313:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829933.576317:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829933.576319:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829933.576322:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3368 00000100:00080000:1.0:1787829933.576326:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787829933.576331:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085941504:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787829933.576332:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829933.576341:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829933.576356:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6940 x1874672085942144/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829933.576365:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787829933.576365:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829933.576370:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829933.576372:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829933.576376:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3368 00000100:00080000:1.0:1787829933.576380:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787829933.576381:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829933/real 1787829933] req@ffff9e35da6f5e00 x1874672085941504/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829958 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829933.576388:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829933.576396:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:0.0:1787829933.576398:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5e00 x1874672085941504/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829958 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829933.576409:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1787829933.576410:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f61c0 x1874672085942272/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829933.576417:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829933.576425:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085941504:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787829933.576430:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829933.576434:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:0.0:1787829933.576436:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085941632:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787829933.576437:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829933.576453:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3368 00000100:00080000:1.0:1787829933.576466:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1787829933.576471:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787829933.576474:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829933.576481:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:0.0:1787829933.576487:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829933/real 1787829933] req@ffff9e35da6f4b40 x1874672085941632/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829958 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829933.576497:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4f00 x1874672085942400/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829933.576504:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4b40 x1874672085941632/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829958 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829933.576521:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829933.576524:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829933.576529:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085941632:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829933.576536:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085941760:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829933.576556:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829933.576567:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829933/real 1787829933] req@ffff9e35da6f70c0 x1874672085941760/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829958 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829933.576578:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f70c0 x1874672085941760/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829958 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829933.576586:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829933.576589:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829933.576592:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085941760:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829933.576598:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085941888:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829933.576614:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829933.576623:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829933/real 1787829933] req@ffff9e35da6f43c0 x1874672085941888/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829958 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829933.576633:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f43c0 x1874672085941888/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829958 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829933.576642:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829933.576644:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829933.576647:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085941888:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829933.576653:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085942016:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829933.576669:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829933.576678:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829933/real 1787829933] req@ffff9e35da6f4000 x1874672085942016/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829958 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829933.576695:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4000 x1874672085942016/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829958 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829933.576703:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829933.576706:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829933.576709:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085942016:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829933.576742:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085942144:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829933.576760:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829933.576770:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829933/real 1787829933] req@ffff9e35da6f6940 x1874672085942144/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829958 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829933.576780:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6940 x1874672085942144/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829958 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829933.576787:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829933.576790:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829933.576793:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085942144:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829933.576818:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085942272:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829933.576836:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829933.576847:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829933/real 1787829933] req@ffff9e35da6f61c0 x1874672085942272/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829958 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829933.576857:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f61c0 x1874672085942272/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829958 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829933.576870:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829933.576873:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829933.576876:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085942272:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829933.576881:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085942400:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829933.576899:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829933.576909:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829933/real 1787829933] req@ffff9e35da6f4f00 x1874672085942400/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829958 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829933.576920:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4f00 x1874672085942400/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829958 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829933.576927:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829933.576930:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829933.576933:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085942400:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829933.703947:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829933.703947:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829933.704109:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829933.704141:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829934.727906:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:0.0:1787829934.727977:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829934.728152:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:3.0:1787829934.728173:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829935.751924:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829935.751924:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:3.0:1787829935.751924:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829935.751997:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829936.775899:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829936.775899:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:3.0:1787829936.775902:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829936.775970:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829937.799881:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:3.0:1787829937.799885:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829937.799886:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:0.0:1787829937.799966:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829938.695893:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829938.695903:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829938.695906:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829938.695915:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3374 00000100:00080000:1.0:1787829938.695924:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829938.695928:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787829938.695965:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6d00 x1874672085942528/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829938.695984:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829938.695986:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829938.695987:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829938.695990:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3374 00000100:00080000:1.0:1787829938.695992:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829938.695994:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829938.696008:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829938.696017:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4780 x1874672085942656/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829938.696024:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787829938.696027:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085942528:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787829938.696027:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829938.696030:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829938.696036:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3374 00000100:00080000:1.0:1787829938.696038:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829938.696040:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829938.696042:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829938.696052:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6580 x1874672085942784/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829938.696058:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829938.696060:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829938.696061:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829938.696064:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3374 00000100:00080000:1.0:1787829938.696074:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829938.696076:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829938.696080:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000800:00000100:0.0:1787829938.696083:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829938.696094:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7480 x1874672085942912/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829938.696101:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829938.696102:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829938.696104:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829938.696106:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3374 00000100:00080000:1.0:1787829938.696109:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829938.696112:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829938.696115:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:0.0:1787829938.696118:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829938/real 1787829938] req@ffff9e35da6f6d00 x1874672085942528/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829963 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829938.696125:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7840 x1874672085943040/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829938.696133:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787829938.696135:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6d00 x1874672085942528/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829963 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829938.696136:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829938.696139:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829938.696146:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3374 00000100:00080000:1.0:1787829938.696149:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829938.696150:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829938.696153:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:0.0:1787829938.696156:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000008:00100000:1.0:1787829938.696158:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829938.696161:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085942528:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1787829938.696169:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4f00 x1874672085943168/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829938.696176:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829938.696178:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829938.696179:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829938.696181:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3374 00000100:00080000:1.0:1787829938.696183:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829938.696187:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787829938.696188:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085942656:192.168.203.141@tcp:38:kworker.0 00000008:00100000:1.0:1787829938.696191:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829938.696204:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f61c0 x1874672085943296/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829938.696214:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1787829938.696219:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787829938.696220:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829938.696224:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829938.696230:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3374 00000100:00080000:1.0:1787829938.696234:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787829938.696236:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829938/real 1787829938] req@ffff9e35da6f4780 x1874672085942656/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829963 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829938.696241:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829938.696250:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:0.0:1787829938.696254:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4780 x1874672085942656/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829963 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829938.696261:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829938.696265:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829938.696267:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6940 x1874672085943424/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829938.696269:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085942656:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829938.696280:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085942784:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787829938.696287:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829938.696290:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829938.696292:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829938.696296:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3374 00000100:00080000:1.0:1787829938.696300:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1787829938.696301:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787829938.696306:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787829938.696312:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829938/real 1787829938] req@ffff9e35da6f6580 x1874672085942784/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829963 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1787829938.696312:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:0.0:1787829938.696323:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6580 x1874672085942784/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829963 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829938.696328:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4000 x1874672085943552/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829938.696355:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829938.696360:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829938.696363:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085942784:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829938.696367:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085942912:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829938.696378:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829938.696384:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829938/real 1787829938] req@ffff9e35da6f7480 x1874672085942912/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829963 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829938.696390:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7480 x1874672085942912/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829963 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829938.696395:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829938.696397:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829938.696399:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085942912:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829938.696402:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085943040:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829938.696415:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829938.696424:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829938/real 1787829938] req@ffff9e35da6f7840 x1874672085943040/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829963 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829938.696436:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7840 x1874672085943040/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829963 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829938.696444:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829938.696448:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829938.696451:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085943040:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829938.696455:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085943168:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829938.696469:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829938.696475:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829938/real 1787829938] req@ffff9e35da6f4f00 x1874672085943168/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829963 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829938.696482:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4f00 x1874672085943168/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829963 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829938.696487:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829938.696489:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829938.696491:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085943168:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829938.696506:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085943296:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829938.696515:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829938.696520:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829938/real 1787829938] req@ffff9e35da6f61c0 x1874672085943296/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829963 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829938.696526:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f61c0 x1874672085943296/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829963 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829938.696531:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829938.696532:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829938.696534:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085943296:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829938.696537:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085943424:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829938.696545:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829938.696551:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829938/real 1787829938] req@ffff9e35da6f6940 x1874672085943424/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829963 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829938.696557:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6940 x1874672085943424/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829963 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829938.696562:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829938.696563:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829938.696565:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085943424:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829938.696568:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085943552:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829938.696575:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829938.696580:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829938/real 1787829938] req@ffff9e35da6f4000 x1874672085943552/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829963 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829938.696587:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4000 x1874672085943552/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829963 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829938.696591:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829938.696593:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829938.696594:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085943552:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829938.823891:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829938.823891:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:3.0:1787829938.823891:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829938.823949:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829939.847911:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829939.847911:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:3.0:1787829939.847911:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829939.847978:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829940.871888:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829940.871996:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829940.872018:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787829940.872072:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829941.895870:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829941.895870:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829941.895870:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787829941.895920:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829942.919890:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829942.919890:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:3.0:1787829942.919895:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829942.919963:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829943.815908:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829943.815917:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829943.815921:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829943.815931:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3379 00000100:00080000:1.0:1787829943.815942:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829943.815948:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787829943.815984:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f43c0 x1874672085943680/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829943.816015:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829943.816018:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829943.816020:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829943.816024:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3379 00000100:00080000:1.0:1787829943.816027:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829943.816030:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829943.816041:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829943.816053:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f70c0 x1874672085943808/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829943.816065:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787829943.816067:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085943680:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787829943.816069:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829943.816073:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829943.816079:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3379 00000100:00080000:1.0:1787829943.816082:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829943.816086:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829943.816091:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829943.816103:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4b40 x1874672085943936/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829943.816112:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829943.816116:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829943.816117:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829943.816121:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3379 00000100:00080000:1.0:1787829943.816126:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829943.816129:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829943.816134:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000800:00000100:0.0:1787829943.816136:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829943.816163:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5e00 x1874672085944064/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829943.816174:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787829943.816179:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829943/real 1787829943] req@ffff9e35da6f43c0 x1874672085943680/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829968 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829943.816179:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829943.816181:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829943.816197:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3379 00000100:00080000:1.0:1787829943.816202:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787829943.816203:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f43c0 x1874672085943680/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829968 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829943.816211:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829943.816219:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00080000:0.0:1787829943.816222:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829943.816229:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829943.816234:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5a40 x1874672085944192/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829943.816235:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085943680:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787829943.816250:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829943.816254:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829943.816255:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829943.816260:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3379 00000100:00080000:1.0:1787829943.816264:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787829943.816266:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085943808:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787829943.816271:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829943.816278:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1787829943.816293:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829943.816299:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f43c0 x1874672085944320/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829943.816309:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829943/real 1787829943] req@ffff9e35da6f70c0 x1874672085943808/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829968 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829943.816312:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829943.816320:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829943.816324:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787829943.816333:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f70c0 x1874672085943808/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829968 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829943.816337:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3379 00000100:00080000:1.0:1787829943.816344:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829943.816345:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829943.816352:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:0.0:1787829943.816353:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829943.816361:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085943808:192.168.203.141@tcp:38:kworker.0 00000008:00100000:1.0:1787829943.816361:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829943.816370:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085943936:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829943.816379:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4000 x1874672085944448/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829943.816390:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1787829943.816394:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787829943.816400:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829943.816403:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787829943.816409:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829943/real 1787829943] req@ffff9e35da6f4b40 x1874672085943936/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829968 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829943.816409:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3379 00000100:00080000:1.0:1787829943.816416:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829943.816424:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787829943.816428:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4b40 x1874672085943936/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829968 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1787829943.816432:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00080000:0.0:1787829943.816447:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829943.816451:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829943.816455:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085943936:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829943.816460:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6940 x1874672085944576/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829943.816464:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085944064:192.168.203.141@tcp:8:kworker.0 00000100:00080000:1.0:1787829943.816480:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829943.816484:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829943.816486:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829943.816490:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3379 00000800:00000100:0.0:1787829943.816500:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787829943.816503:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829943.816510:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787829943.816514:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829943/real 1787829943] req@ffff9e35da6f5e00 x1874672085944064/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829968 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1787829943.816517:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:1.0:1787829943.816533:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f61c0 x1874672085944704/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829943.816534:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5e00 x1874672085944064/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829968 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829943.816560:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829943.816564:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829943.816568:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085944064:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829943.816574:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085944192:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829943.816595:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829943.816606:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829943/real 1787829943] req@ffff9e35da6f5a40 x1874672085944192/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829968 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829943.816617:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5a40 x1874672085944192/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829968 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829943.816625:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829943.816628:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829943.816632:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085944192:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829943.816653:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085944320:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829943.816689:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829943.816701:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829943/real 1787829943] req@ffff9e35da6f43c0 x1874672085944320/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829968 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829943.816724:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f43c0 x1874672085944320/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829968 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829943.816732:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829943.816735:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829943.816738:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085944320:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829943.816744:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085944448:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829943.816761:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829943.816771:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829943/real 1787829943] req@ffff9e35da6f4000 x1874672085944448/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829968 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829943.816782:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4000 x1874672085944448/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829968 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829943.816789:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829943.816792:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829943.816795:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085944448:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829943.816830:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085944576:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829943.816849:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829943.816861:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829943/real 1787829943] req@ffff9e35da6f6940 x1874672085944576/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829968 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829943.816873:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6940 x1874672085944576/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829968 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829943.816881:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829943.816884:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829943.816887:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085944576:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829943.816893:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085944704:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829943.816910:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829943.816919:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829943/real 1787829943] req@ffff9e35da6f61c0 x1874672085944704/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829968 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829943.816930:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f61c0 x1874672085944704/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829968 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829943.816938:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829943.816941:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829943.816945:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085944704:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829943.943889:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:1.0:1787829943.943941:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829943.943996:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:3.0:1787829943.944010:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829944.967885:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829944.967888:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:3.0:1787829944.967888:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787829944.967965:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829945.991884:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829945.991921:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829945.991921:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829945.991972:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829947.015907:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829947.015907:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829947.015907:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787829947.015955:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829948.039893:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829948.039893:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829948.039912:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787829948.039950:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829948.935905:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829948.935913:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829948.935917:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829948.935926:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3384 00000100:00080000:1.0:1787829948.935936:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829948.935942:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787829948.935975:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4f00 x1874672085944832/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829948.935998:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829948.936001:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829948.936002:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829948.936005:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3384 00000100:00080000:1.0:1787829948.936008:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829948.936010:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829948.936020:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829948.936033:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7840 x1874672085944960/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829948.936043:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829948.936047:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829948.936049:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829948.936052:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3384 00000100:00080000:1.0:1787829948.936055:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829948.936060:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787829948.936065:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085944832:192.168.203.141@tcp:250:kworker.0 00000008:00100000:1.0:1787829948.936065:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829948.936077:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7480 x1874672085945088/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829948.936086:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829948.936088:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829948.936089:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829948.936092:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3384 00000100:00080000:1.0:1787829948.936094:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829948.936097:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829948.936100:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:1.0:1787829948.936109:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6580 x1874672085945216/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829948.936115:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829948.936117:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829948.936118:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829948.936121:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3384 00000100:00080000:1.0:1787829948.936135:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829948.936138:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829948.936142:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000800:00000100:0.0:1787829948.936149:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787829948.936153:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4780 x1874672085945344/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829948.936161:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829948.936164:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829948.936166:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829948.936169:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3384 00000100:00080000:1.0:1787829948.936174:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829948.936176:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829948.936179:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829948.936185:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6d00 x1874672085945472/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829948.936191:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829948.936194:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829948.936195:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829948.936198:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3384 00000100:00080000:1.0:1787829948.936200:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829948.936202:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829948.936205:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829948.936214:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829948/real 1787829948] req@ffff9e35da6f4f00 x1874672085944832/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829973 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829948.936215:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f61c0 x1874672085945600/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829948.936235:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829948.936243:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829948.936245:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829948.936249:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3384 00000100:00080000:1.0:1787829948.936252:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829948.936256:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787829948.936260:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4f00 x1874672085944832/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829973 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1787829948.936261:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:1.0:1787829948.936272:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6940 x1874672085945728/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829948.936282:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829948.936282:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829948.936290:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829948.936292:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829948.936294:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:1.0:1787829948.936298:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3384 00000100:00080000:1.0:1787829948.936302:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787829948.936306:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085944832:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787829948.936307:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829948.936313:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:0.0:1787829948.936323:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085944960:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829948.936324:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4000 x1874672085945856/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829948.936364:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829948.936382:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829948/real 1787829948] req@ffff9e35da6f7840 x1874672085944960/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829973 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829948.936398:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7840 x1874672085944960/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829973 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829948.936409:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829948.936413:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829948.936417:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085944960:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829948.936450:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085945088:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829948.936474:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829948.936487:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829948/real 1787829948] req@ffff9e35da6f7480 x1874672085945088/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829973 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829948.936510:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7480 x1874672085945088/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829973 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829948.936519:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829948.936523:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829948.936527:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085945088:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829948.936534:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085945216:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829948.936557:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829948.936584:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829948/real 1787829948] req@ffff9e35da6f6580 x1874672085945216/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829973 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829948.936599:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6580 x1874672085945216/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829973 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829948.936609:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829948.936613:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829948.936616:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085945216:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829948.936640:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085945344:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829948.936674:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829948.936688:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829948/real 1787829948] req@ffff9e35da6f4780 x1874672085945344/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829973 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829948.936701:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4780 x1874672085945344/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829973 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829948.936718:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829948.936722:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829948.936725:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085945344:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829948.936732:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085945472:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829948.936754:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829948.936768:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829948/real 1787829948] req@ffff9e35da6f6d00 x1874672085945472/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829973 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829948.936782:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6d00 x1874672085945472/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829973 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829948.936792:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829948.936795:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829948.936831:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085945472:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829948.936839:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085945600:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829948.936863:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829948.936877:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829948/real 1787829948] req@ffff9e35da6f61c0 x1874672085945600/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829973 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829948.936891:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f61c0 x1874672085945600/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829973 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829948.936900:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829948.936904:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829948.936908:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085945600:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829948.936915:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085945728:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829948.936936:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829948.936948:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829948/real 1787829948] req@ffff9e35da6f6940 x1874672085945728/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829973 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829948.936961:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6940 x1874672085945728/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829973 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829948.936971:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829948.936974:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829948.936978:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085945728:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829948.936985:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085945856:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829948.937007:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829948.937021:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829948/real 1787829948] req@ffff9e35da6f4000 x1874672085945856/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829973 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829948.937035:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4000 x1874672085945856/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829973 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829948.937045:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829948.937049:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829948.937053:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085945856:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829949.063899:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829949.063899:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829949.063901:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787829949.063948:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829950.087932:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829950.087932:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829950.087962:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787829950.087976:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829951.111898:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829951.111941:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829951.111942:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:1.0:1787829951.112022:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829952.135918:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829952.135941:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829952.135967:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:0.0:1787829952.135973:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829953.159877:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829953.159877:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829953.159907:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787829953.159988:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829954.055936:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829954.055945:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829954.055949:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829954.055959:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3389 00000100:00080000:2.0:1787829954.055970:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829954.055976:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787829954.056012:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cb40 x1874672085945984/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829954.056040:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829954.056044:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829954.056046:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829954.056050:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3389 00000100:00080000:2.0:1787829954.056053:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829954.056058:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829954.056068:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829954.056081:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085945984:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1787829954.056085:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498fc00 x1874672085946112/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829954.056094:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829954.056096:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829954.056097:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829954.056100:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3389 00000100:00080000:2.0:1787829954.056103:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829954.056106:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829954.056109:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829954.056117:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498ed00 x1874672085946240/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829954.056123:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829954.056125:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829954.056126:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829954.056128:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3389 00000100:00080000:2.0:1787829954.056131:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829954.056133:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829954.056136:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000800:00000100:0.0:1787829954.056142:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829954.056146:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c000 x1874672085946368/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829954.056153:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829954.056156:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829954.056157:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829954.056160:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3389 00000100:00080000:2.0:1787829954.056162:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829954.056164:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829954.056167:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:2.0:1787829954.056173:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498de00 x1874672085946496/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829954.056179:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787829954.056183:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829954/real 1787829954] req@ffff9e35c498cb40 x1874672085945984/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829979 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829954.056184:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829954.056187:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829954.056194:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3389 00000100:00080000:2.0:1787829954.056198:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829954.056201:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829954.056204:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829954.056208:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498cb40 x1874672085945984/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829979 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829954.056212:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e940 x1874672085946624/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829954.056238:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829954.056239:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829954.056245:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829954.056246:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829954.056248:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:2.0:1787829954.056251:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3389 00000100:00080000:2.0:1787829954.056255:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787829954.056258:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085945984:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1787829954.056259:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829954.056264:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829954.056274:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498d680 x1874672085946752/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829954.056279:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829954.056282:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829954.056283:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829954.056285:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3389 00000100:00080000:2.0:1787829954.056287:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829954.056289:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829954.056292:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:0.0:1787829954.056295:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085946112:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829954.056300:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cb40 x1874672085946880/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829954.056307:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829954.056309:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829954.056310:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829954.056313:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3389 00000100:00080000:2.0:1787829954.056317:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829954.056320:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829954.056323:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000800:00000100:0.0:1787829954.056325:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829954.056333:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e580 x1874672085947008/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829954.056342:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829954/real 1787829954] req@ffff9e35c498fc00 x1874672085946112/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829979 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829954.056357:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498fc00 x1874672085946112/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829979 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829954.056366:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829954.056370:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829954.056373:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085946112:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829954.056379:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085946240:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829954.056397:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829954.056405:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829954/real 1787829954] req@ffff9e35c498ed00 x1874672085946240/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829979 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829954.056416:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498ed00 x1874672085946240/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829979 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829954.056424:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829954.056426:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829954.056429:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085946240:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829954.056434:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085946368:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829954.056447:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829954.056455:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829954/real 1787829954] req@ffff9e35c498c000 x1874672085946368/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829979 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829954.056464:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c000 x1874672085946368/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829979 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829954.056472:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829954.056475:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829954.056478:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085946368:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829954.056484:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085946496:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829954.056499:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829954.056505:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829954/real 1787829954] req@ffff9e35c498de00 x1874672085946496/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829979 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829954.056522:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498de00 x1874672085946496/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829979 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829954.056529:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829954.056531:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829954.056533:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085946496:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829954.056536:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085946624:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829954.056549:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829954.056555:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829954/real 1787829954] req@ffff9e35c498e940 x1874672085946624/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829979 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829954.056562:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e940 x1874672085946624/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829979 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829954.056567:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829954.056570:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829954.056573:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085946624:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829954.056578:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085946752:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829954.056615:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829954.056626:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829954/real 1787829954] req@ffff9e35c498d680 x1874672085946752/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829979 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829954.056644:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498d680 x1874672085946752/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829979 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829954.056658:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829954.056661:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829954.056663:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085946752:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829954.056687:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085946880:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829954.056707:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829954.056726:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829954/real 1787829954] req@ffff9e35c498cb40 x1874672085946880/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829979 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829954.056739:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498cb40 x1874672085946880/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829979 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829954.056747:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829954.056750:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829954.056753:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085946880:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829954.056759:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085947008:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829954.056777:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829954.056788:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829954/real 1787829954] req@ffff9e35c498e580 x1874672085947008/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829979 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829954.056821:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e580 x1874672085947008/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829979 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829954.056829:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829954.056832:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829954.056835:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085947008:192.168.203.141@tcp:8:kworker.0 00010000:00010000:0.0:1787829954.183957:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829954.183977:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829954.184043:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829954.184045:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:3.0:1787829955.207935:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829955.207937:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829955.207946:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:0.0:1787829955.208064:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829956.231895:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829956.232017:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829956.232038:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787829956.232072:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829957.255915:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:0.0:1787829957.255965:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829957.256023:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:3.0:1787829957.256044:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829958.279944:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829958.279982:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829958.280117:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:1.0:1787829958.280132:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829959.175974:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829959.175982:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787829959.175985:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829959.175993:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3394 00000100:00080000:3.0:1787829959.176003:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829959.176010:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787829959.176047:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a580 x1874672085947136/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829959.176073:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829959.176078:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829959.176079:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829959.176083:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3394 00000100:00080000:3.0:1787829959.176087:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829959.176092:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829959.176102:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829959.176120:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499e00 x1874672085947264/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829959.176129:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085947136:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1787829959.176134:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829959.176137:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787829959.176139:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829959.176145:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3394 00000100:00080000:3.0:1787829959.176150:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829959.176154:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829959.176158:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829959.176171:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b0c0 x1874672085947392/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829959.176188:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829959.176191:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787829959.176192:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829959.176196:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3394 00000100:00080000:3.0:1787829959.176199:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829959.176203:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829959.176208:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:3.0:1787829959.176219:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a940 x1874672085947520/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829959.176228:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1787829959.176229:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1787829959.176234:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787829959.176236:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829959.176241:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3394 00000100:00080000:3.0:1787829959.176244:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829959.176247:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829959.176252:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:3.0:1787829959.176267:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499680 x1874672085947648/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829959.176275:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829959.176278:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829959.176280:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787829959.176286:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829959/real 1787829959] req@ffff9e35c649a580 x1874672085947136/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829984 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829959.176287:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3394 00000100:00080000:3.0:1787829959.176293:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829959.176300:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829959.176307:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829959.176314:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a580 x1874672085947136/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829984 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829959.176319:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b840 x1874672085947776/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829959.176331:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829959.176336:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787829959.176339:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829959.176341:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829959.176344:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3394 00000100:00080000:0.0:1787829959.176350:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:3.0:1787829959.176350:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829959.176357:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787829959.176359:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085947136:192.168.203.141@tcp:250:kworker.0 00000008:00100000:3.0:1787829959.176364:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829959.176378:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649bc00 x1874672085947904/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829959.176386:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787829959.176387:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085947264:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1787829959.176390:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787829959.176393:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829959.176397:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3394 00000100:00080000:3.0:1787829959.176400:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829959.176402:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829959.176406:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:3.0:1787829959.176414:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498b40 x1874672085948032/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829959.176422:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1787829959.176423:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829959.176428:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787829959.176429:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829959.176433:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3394 00000100:00080000:3.0:1787829959.176435:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829959.176437:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829959.176440:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:0.0:1787829959.176444:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829959/real 1787829959] req@ffff9e35c6499e00 x1874672085947264/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829984 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829959.176449:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a1c0 x1874672085948160/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829959.176477:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499e00 x1874672085947264/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829984 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829959.176488:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829959.176492:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829959.176495:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085947264:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829959.176502:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085947392:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829959.176521:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829959.176541:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829959/real 1787829959] req@ffff9e35c649b0c0 x1874672085947392/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829984 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829959.176567:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b0c0 x1874672085947392/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829984 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829959.176575:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829959.176579:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829959.176582:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085947392:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829959.176588:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085947520:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829959.176613:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829959.176623:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829959/real 1787829959] req@ffff9e35c649a940 x1874672085947520/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829984 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829959.176634:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a940 x1874672085947520/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829984 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829959.176641:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829959.176644:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829959.176647:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085947520:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829959.176652:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085947648:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829959.176670:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829959.176679:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829959/real 1787829959] req@ffff9e35c6499680 x1874672085947648/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829984 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829959.176689:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499680 x1874672085947648/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829984 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829959.176703:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829959.176706:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829959.176709:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085947648:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829959.176715:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085947776:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829959.176732:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829959.176742:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829959/real 1787829959] req@ffff9e35c649b840 x1874672085947776/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829984 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829959.176752:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b840 x1874672085947776/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829984 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829959.176760:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829959.176763:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829959.176766:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085947776:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829959.176785:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085947904:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829959.176839:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829959.176851:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829959/real 1787829959] req@ffff9e35c649bc00 x1874672085947904/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829984 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829959.176871:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649bc00 x1874672085947904/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829984 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829959.176879:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829959.176882:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829959.176885:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085947904:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829959.176890:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085948032:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829959.176916:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829959.176926:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829959/real 1787829959] req@ffff9e35c6498b40 x1874672085948032/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829984 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829959.176936:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498b40 x1874672085948032/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829984 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829959.176944:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829959.176954:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829959.176958:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085948032:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829959.176963:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085948160:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829959.176980:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829959.176990:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829959/real 1787829959] req@ffff9e35c649a1c0 x1874672085948160/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829984 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829959.177000:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a1c0 x1874672085948160/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829984 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829959.177015:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829959.177019:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829959.177022:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085948160:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829959.303904:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829959.303910:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829959.303928:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787829959.303962:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829960.327886:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829960.327902:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829960.327928:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829960.328014:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829961.351928:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829961.351930:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829961.351956:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:1.0:1787829961.352663:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829962.375932:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829962.376404:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:3.0:1787829962.376412:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787829962.376915:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829963.399869:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:3.0:1787829963.399884:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829963.399887:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829963.399941:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829964.296545:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829964.296554:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787829964.296558:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829964.296568:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3399 00000100:00080000:0.0:1787829964.296580:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829964.296587:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787829964.296621:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718b40 x1874672085948288/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829964.296647:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829964.296651:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829964.296652:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829964.296656:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3399 00000100:00080000:0.0:1787829964.296660:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829964.296664:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829964.296675:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829964.296688:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a580 x1874672085948416/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829964.296699:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829964.296702:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787829964.296704:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829964.296708:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3399 00000100:00080000:0.0:1787829964.296711:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829964.296715:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829964.296719:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829964.296730:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718000 x1874672085948544/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829964.296740:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829964.296743:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829964.296745:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829964.296754:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3399 00000100:00080000:0.0:1787829964.296757:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829964.296761:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829964.296766:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:0.0:1787829964.296777:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7183c0 x1874672085948672/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829964.296786:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829964.296790:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829964.296791:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829964.296796:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3399 00000100:00080000:0.0:1787829964.296818:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829964.296822:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829964.296827:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:0.0:1787829964.296837:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da719a40 x1874672085948800/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829964.296846:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829964.296849:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829964.296851:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829964.296854:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3399 00000100:00080000:0.0:1787829964.296857:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829964.296861:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829964.296865:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829964.296876:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718780 x1874672085948928/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829964.296884:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829964.296887:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787829964.296889:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829964.296892:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3399 00000100:00080000:0.0:1787829964.296895:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829964.296898:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829964.296903:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829964.296912:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a940 x1874672085949056/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829964.296921:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829964.296924:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829964.296925:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829964.296929:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3399 00000100:00080000:0.0:1787829964.296932:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829964.296936:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829964.296940:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:0.0:1787829964.296950:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718f00 x1874672085949184/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829964.296958:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829964.296961:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829964.296963:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829964.296966:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3399 00000100:00080000:0.0:1787829964.296969:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829964.296972:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829964.296977:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:0.0:1787829964.296987:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7192c0 x1874672085949312/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829964.297126:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085948288:192.168.203.141@tcp:250:kworker.0 00000800:00000100:0.0:1787829964.297186:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829964.297221:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829964/real 1787829964] req@ffff9e35da718b40 x1874672085948288/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829989 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829964.297237:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718b40 x1874672085948288/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829989 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829964.297247:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829964.297251:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829964.297255:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085948288:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1787829964.297262:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085948416:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829964.297282:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829964.297294:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829964/real 1787829964] req@ffff9e35da71a580 x1874672085948416/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829989 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829964.297306:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672085948416/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829989 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829964.297314:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829964.297317:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829964.297320:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085948416:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829964.297325:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085948544:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829964.297344:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829964.297354:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829964/real 1787829964] req@ffff9e35da718000 x1874672085948544/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829989 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829964.297364:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718000 x1874672085948544/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829989 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829964.297376:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829964.297380:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829964.297384:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085948544:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829964.297389:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085948672:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829964.297408:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829964.297418:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829964/real 1787829964] req@ffff9e35da7183c0 x1874672085948672/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829989 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829964.297428:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672085948672/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829989 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829964.297436:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829964.297440:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829964.297443:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085948672:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829964.297450:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085948800:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829964.297468:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829964.297480:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829964/real 1787829964] req@ffff9e35da719a40 x1874672085948800/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829989 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829964.297492:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672085948800/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829989 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829964.297500:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829964.297503:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829964.297507:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085948800:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829964.297862:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085948928:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829964.297890:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829964.297905:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829964/real 1787829964] req@ffff9e35da718780 x1874672085948928/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829989 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829964.297918:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718780 x1874672085948928/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829989 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829964.297927:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829964.297930:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829964.297933:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085948928:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829964.297938:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085949056:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829964.297958:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829964.297968:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829964/real 1787829964] req@ffff9e35da71a940 x1874672085949056/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829989 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829964.297979:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672085949056/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829989 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829964.297986:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829964.297990:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829964.297993:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085949056:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829964.297998:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085949184:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829964.298017:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829964.298028:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829964/real 1787829964] req@ffff9e35da718f00 x1874672085949184/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829989 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829964.298038:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672085949184/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829989 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829964.298046:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829964.298050:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829964.298053:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085949184:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829964.298058:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085949312:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829964.298078:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829964.298088:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829964/real 1787829964] req@ffff9e35da7192c0 x1874672085949312/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829989 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829964.298100:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7192c0 x1874672085949312/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829989 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829964.298108:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829964.298111:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829964.298114:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085949312:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829964.423905:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829964.423909:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829964.424009:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787829964.424066:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829965.447909:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829965.447909:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829965.447933:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787829965.448143:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829966.471942:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829966.471963:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829966.471988:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829966.473788:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829967.495930:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829967.495957:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:0.0:1787829967.496007:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829967.496048:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829968.520004:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829968.520010:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:3.0:1787829968.521592:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:0.0:1787829968.521937:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829969.416580:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829969.416590:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829969.416594:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829969.416604:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3404 00000100:00080000:1.0:1787829969.416615:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829969.416623:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787829969.416663:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f43c0 x1874672085949440/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829969.416689:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829969.416693:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829969.416695:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829969.416700:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3404 00000100:00080000:1.0:1787829969.416704:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829969.416709:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829969.416720:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829969.416728:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085949440:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1787829969.416735:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5a40 x1874672085949568/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829969.416748:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829969.416752:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829969.416753:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829969.416758:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3404 00000100:00080000:1.0:1787829969.416762:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829969.416765:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829969.416770:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829969.416784:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5e00 x1874672085949696/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829969.416795:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787829969.416795:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829969.416820:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829969.416825:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829969.416830:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3404 00000100:00080000:1.0:1787829969.416835:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829969.416839:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829969.416844:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:1.0:1787829969.416857:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4b40 x1874672085949824/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829969.416866:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829969/real 1787829969] req@ffff9e35da6f43c0 x1874672085949440/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829994 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829969.416869:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829969.416878:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829969.416882:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829969.416889:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3404 00000100:00080000:1.0:1787829969.416894:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787829969.416900:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f43c0 x1874672085949440/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829994 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829969.416900:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829969.416908:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00080000:0.0:1787829969.416919:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829969.416925:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787829969.416925:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f70c0 x1874672085949952/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829969.416933:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085949440:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787829969.416944:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829969.416948:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829969.416950:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829969.416955:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3404 00000100:00080000:1.0:1787829969.416959:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787829969.416965:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085949568:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787829969.416966:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829969.416975:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829969.416990:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f43c0 x1874672085950080/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829969.416999:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787829969.417003:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829969.417008:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829969.417010:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787829969.417015:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829969/real 1787829969] req@ffff9e35da6f5a40 x1874672085949568/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829994 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829969.417016:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3404 00000100:00080000:1.0:1787829969.417022:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829969.417026:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829969.417030:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829969.417045:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4000 x1874672085950208/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829969.417048:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5a40 x1874672085949568/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829994 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829969.417063:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829969.417070:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829969.417071:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829969.417072:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829969.417078:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:1.0:1787829969.417081:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3404 00000100:00100000:0.0:1787829969.417086:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085949568:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787829969.417088:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829969.417096:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829969.417102:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:0.0:1787829969.417108:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085949696:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787829969.417115:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6940 x1874672085950336/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829969.417124:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829969.417127:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829969.417129:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829969.417133:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3404 00000800:00000100:0.0:1787829969.417137:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787829969.417138:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829969.417144:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829969.417149:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:0.0:1787829969.417154:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829969/real 1787829969] req@ffff9e35da6f5e00 x1874672085949696/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829994 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829969.417163:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f61c0 x1874672085950464/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829969.417201:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5e00 x1874672085949696/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829994 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829969.417210:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829969.417214:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829969.417219:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085949696:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829969.417225:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085949824:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829969.417245:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829969.417256:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829969/real 1787829969] req@ffff9e35da6f4b40 x1874672085949824/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829994 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829969.417266:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4b40 x1874672085949824/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829994 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829969.417273:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829969.417276:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829969.417279:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085949824:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829969.417284:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085949952:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829969.417500:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829969.417514:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829969/real 1787829969] req@ffff9e35da6f70c0 x1874672085949952/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829994 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829969.417526:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f70c0 x1874672085949952/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829994 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829969.417541:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829969.417545:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829969.417548:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085949952:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829969.417569:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085950080:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829969.417589:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829969.417638:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829969/real 1787829969] req@ffff9e35da6f43c0 x1874672085950080/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829994 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829969.417650:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f43c0 x1874672085950080/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829994 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829969.417658:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829969.417661:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829969.417664:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085950080:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829969.417669:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085950208:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829969.417688:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829969.417700:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829969/real 1787829969] req@ffff9e35da6f4000 x1874672085950208/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829994 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829969.417711:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4000 x1874672085950208/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829994 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829969.417719:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829969.417723:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829969.417726:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085950208:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829969.417732:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085950336:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829969.417750:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829969.417761:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829969/real 1787829969] req@ffff9e35da6f6940 x1874672085950336/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829994 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829969.417771:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6940 x1874672085950336/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829994 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829969.417779:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829969.417782:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829969.417785:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085950336:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829969.417789:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085950464:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829969.417827:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829969.417839:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829969/real 1787829969] req@ffff9e35da6f61c0 x1874672085950464/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829994 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829969.417850:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f61c0 x1874672085950464/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829994 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829969.417858:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829969.417861:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829969.417864:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085950464:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829969.543918:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829969.543935:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829969.543948:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829969.543974:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829970.567909:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829970.567909:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829970.567915:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787829970.568054:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829971.591991:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829971.591997:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829971.591997:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829971.592023:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829972.615916:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829972.615932:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829972.615990:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829972.615999:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829973.639955:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829973.640032:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829973.640193:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:0.0:1787829973.640213:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829974.535948:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829974.535957:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787829974.535961:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829974.535971:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3409 00000100:00080000:0.0:1787829974.535982:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829974.535989:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787829974.536024:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7192c0 x1874672085950592/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829974.536056:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829974.536060:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829974.536061:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829974.536065:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3409 00000100:00080000:0.0:1787829974.536068:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829974.536072:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829974.536083:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829974.536095:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085950592:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1787829974.536097:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718f00 x1874672085950720/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829974.536110:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829974.536114:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787829974.536116:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829974.536120:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3409 00000100:00080000:0.0:1787829974.536124:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829974.536128:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829974.536133:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829974.536145:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a940 x1874672085950848/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829974.536152:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787829974.536157:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829974.536161:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829974.536163:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829974.536168:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3409 00000100:00080000:0.0:1787829974.536172:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829974.536176:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829974.536182:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:2.0:1787829974.536189:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829974/real 1787829974] req@ffff9e35da7192c0 x1874672085950592/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829999 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829974.536194:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718780 x1874672085950976/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829974.536204:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829974.536210:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829974.536214:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829974.536222:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3409 00000100:00080000:0.0:1787829974.536226:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829974.536229:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829974.536234:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:0.0:1787829974.536244:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da719a40 x1874672085951104/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829974.536259:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829974.536263:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:2.0:1787829974.536266:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7192c0 x1874672085950592/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829999 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829974.536267:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829974.536279:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829974.536283:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3409 00000100:00080000:0.0:1787829974.536289:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829974.536289:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:0.0:1787829974.536294:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787829974.536297:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085950592:192.168.203.141@tcp:250:kworker.0 00000008:00100000:0.0:1787829974.536302:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829974.536315:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7183c0 x1874672085951232/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829974.536324:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829974.536328:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:2.0:1787829974.536330:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085950720:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787829974.536331:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829974.536338:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3409 00000100:00080000:0.0:1787829974.536342:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829974.536346:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829974.536350:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1787829974.536354:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829974.536365:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718000 x1874672085951360/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829974.536366:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829974/real 1787829974] req@ffff9e35da718f00 x1874672085950720/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829999 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829974.536395:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672085950720/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829999 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829974.536397:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829974.536406:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829974.536408:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829974.536409:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829974.536413:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:0.0:1787829974.536419:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3409 00000100:00100000:2.0:1787829974.536419:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085950720:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829974.536426:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085950848:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787829974.536427:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829974.536435:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787829974.536441:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000800:00000100:2.0:1787829974.536460:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829974.536469:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a580 x1874672085951488/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829974.536473:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829974/real 1787829974] req@ffff9e35da71a940 x1874672085950848/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829999 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829974.536492:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829974.536496:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829974.536498:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787829974.536501:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672085950848/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829999 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829974.536504:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3409 00000100:00080000:0.0:1787829974.536511:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829974.536517:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829974.536518:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:2.0:1787829974.536525:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000008:00100000:0.0:1787829974.536526:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:2.0:1787829974.536533:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085950848:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829974.536541:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085950976:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829974.536544:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718b40 x1874672085951616/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829974.536573:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829974.536591:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829974/real 1787829974] req@ffff9e35da718780 x1874672085950976/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829999 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829974.536603:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718780 x1874672085950976/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829999 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829974.536611:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829974.536614:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829974.536617:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085950976:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829974.536621:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085951104:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829974.536635:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829974.536641:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829974/real 1787829974] req@ffff9e35da719a40 x1874672085951104/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829999 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829974.536648:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672085951104/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829999 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829974.536654:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829974.536656:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829974.536667:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085951104:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829974.536672:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085951232:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787829974.536690:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829974.536701:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829974/real 1787829974] req@ffff9e35da7183c0 x1874672085951232/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829999 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829974.536712:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672085951232/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829999 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829974.536721:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829974.536724:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829974.536728:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085951232:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829974.536748:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085951360:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787829974.536781:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829974.536793:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829974/real 1787829974] req@ffff9e35da718000 x1874672085951360/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829999 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829974.536836:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718000 x1874672085951360/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829999 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829974.536844:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829974.536847:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829974.536850:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085951360:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787829974.536856:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085951488:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829974.536875:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829974.536886:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829974/real 1787829974] req@ffff9e35da71a580 x1874672085951488/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829999 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829974.536908:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672085951488/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829999 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829974.536917:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829974.536920:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829974.536923:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085951488:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829974.536929:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085951616:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787829974.536947:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787829974.536957:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829974/real 1787829974] req@ffff9e35da718b40 x1874672085951616/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829999 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829974.536969:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718b40 x1874672085951616/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829999 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829974.536977:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829974.536980:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787829974.536983:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085951616:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787829974.663923:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:3.0:1787829974.663923:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829974.663930:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787829974.664011:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829975.687911:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829975.687911:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829975.687953:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829975.688453:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829976.711919:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829976.711953:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787829976.712211:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829976.712258:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829977.735914:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787829977.735991:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829977.737474:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829977.737602:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829978.759920:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829978.759993:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829978.760023:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829978.760124:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00080000:2.0:1787829979.655963:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829979.655973:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829979.655977:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829979.655986:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3415 00000100:00080000:2.0:1787829979.655998:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829979.656005:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787829979.656040:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cf00 x1874672085951744/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829979.657467:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085951744:192.168.203.141@tcp:250:kworker.0 00000800:00000100:0.0:1787829979.657533:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1787829979.657589:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829979.657594:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829979.657596:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829979.657601:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3415 00000100:00080000:2.0:1787829979.657605:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829979.657610:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829979.657620:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829979.657636:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f0c0 x1874672085951872/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829979.657696:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829979/real 1787829979] req@ffff9e35c498cf00 x1874672085951744/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830004 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829979.657712:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498cf00 x1874672085951744/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830004 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829979.657722:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829979.657726:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829979.657729:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085951744:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1787829979.657750:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085951872:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829979.657771:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829979.657784:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829979/real 1787829979] req@ffff9e35c498f0c0 x1874672085951872/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830004 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829979.657795:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f0c0 x1874672085951872/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830004 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829979.657833:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829979.657837:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829979.657840:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085951872:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1787829979.658313:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829979.658318:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829979.658320:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829979.658324:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3415 00000100:00080000:2.0:1787829979.658328:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829979.658333:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829979.658339:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829979.658351:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f840 x1874672085952000/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829979.658390:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085952000:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829979.658423:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1787829979.658445:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829979.658449:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829979.658451:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829979.658455:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3415 00000100:00080000:2.0:1787829979.658459:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829979.658463:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829979.658468:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:2.0:1787829979.658479:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c3c0 x1874672085952128/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829979.658518:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829979/real 1787829979] req@ffff9e35c498f840 x1874672085952000/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830004 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829979.658530:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f840 x1874672085952000/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830004 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829979.658537:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829979.658540:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829979.658543:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085952000:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829979.658556:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085952128:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829979.658576:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829979.658587:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829979/real 1787829979] req@ffff9e35c498c3c0 x1874672085952128/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830004 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829979.658598:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c3c0 x1874672085952128/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830004 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829979.658607:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829979.658609:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829979.658612:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085952128:192.168.203.141@tcp:8:kworker.0 00000100:00080000:2.0:1787829979.658697:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829979.658702:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829979.658704:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829979.658708:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3415 00000100:00080000:2.0:1787829979.658713:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829979.658717:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829979.658722:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:2.0:1787829979.658734:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c780 x1874672085952256/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829979.658777:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085952256:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829979.658813:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1787829979.658842:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829979.658846:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829979.658848:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829979.658852:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3415 00000100:00080000:2.0:1787829979.658856:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829979.658860:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829979.658865:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829979.658877:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f480 x1874672085952384/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829979.658922:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829979/real 1787829979] req@ffff9e35c498c780 x1874672085952256/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830004 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829979.658935:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c780 x1874672085952256/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830004 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829979.658943:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829979.658946:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829979.658949:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085952256:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829979.658962:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085952384:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829979.658983:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829979.658994:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829979/real 1787829979] req@ffff9e35c498f480 x1874672085952384/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830004 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829979.659003:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f480 x1874672085952384/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830004 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829979.659010:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829979.659013:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829979.659015:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085952384:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1787829979.659097:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829979.659101:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829979.659102:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829979.659107:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3415 00000100:00080000:2.0:1787829979.659111:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829979.659115:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829979.659120:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829979.659132:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e1c0 x1874672085952512/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829979.659173:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085952512:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829979.659195:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1787829979.659216:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829979.659220:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829979.659221:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829979.659225:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3415 00000100:00080000:2.0:1787829979.659229:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829979.659233:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829979.659239:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:2.0:1787829979.659251:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f480 x1874672085952640/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829979.659288:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829979/real 1787829979] req@ffff9e35c498e1c0 x1874672085952512/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830004 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829979.659299:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e1c0 x1874672085952512/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830004 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829979.659306:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829979.659309:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829979.659312:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085952512:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829979.659324:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085952640:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829979.659343:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829979.659353:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829979/real 1787829979] req@ffff9e35c498f480 x1874672085952640/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830004 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829979.659364:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f480 x1874672085952640/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830004 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829979.659376:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829979.659378:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829979.659381:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085952640:192.168.203.141@tcp:8:kworker.0 00000100:00080000:2.0:1787829979.659472:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829979.659476:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829979.659477:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829979.659482:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3415 00000100:00080000:2.0:1787829979.659486:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829979.659490:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829979.659495:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:2.0:1787829979.659507:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c780 x1874672085952768/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829979.659551:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085952768:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829979.659573:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829979.659637:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829979/real 1787829979] req@ffff9e35c498c780 x1874672085952768/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830004 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829979.659650:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c780 x1874672085952768/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830004 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829979.659658:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829979.659660:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829979.659663:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085952768:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829979.783904:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829979.784040:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829979.785568:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:0.0:1787829979.785951:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829980.807887:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787829980.808124:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829980.810837:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829980.810854:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829981.831901:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829981.831901:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829981.831997:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829981.834933:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829982.855933:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787829982.856068:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829982.856095:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829982.856652:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829983.880004:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829983.880007:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787829983.880011:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787829983.880174:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829984.776085:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829984.776097:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787829984.776102:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829984.776114:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3420 00000100:00080000:3.0:1787829984.776126:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829984.776135:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787829984.776190:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a1c0 x1874672085952896/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829984.776224:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829984.776228:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829984.776229:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829984.776234:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3420 00000100:00080000:3.0:1787829984.776238:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829984.776242:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829984.776253:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829984.776266:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498b40 x1874672085953024/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829984.776280:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085952896:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1787829984.776280:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829984.776289:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787829984.776291:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829984.776295:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3420 00000100:00080000:3.0:1787829984.776298:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829984.776302:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829984.776308:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829984.776320:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649bc00 x1874672085953152/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829984.776329:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829984.776333:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787829984.776335:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829984.776340:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3420 00000100:00080000:3.0:1787829984.776344:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829984.776348:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829984.776353:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:3.0:1787829984.776366:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b840 x1874672085953280/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829984.776390:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829984.776394:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787829984.776396:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829984.776401:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3420 00000100:00080000:3.0:1787829984.776405:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829984.776410:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829984.776416:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:0.0:1787829984.776428:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085953024:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787829984.776431:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499680 x1874672085953408/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829984.776444:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829984.776449:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829984.776451:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829984.776456:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3420 00000100:00080000:3.0:1787829984.776461:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829984.776464:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787829984.776467:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085953152:192.168.203.141@tcp:38:kworker.0 00000008:00100000:3.0:1787829984.776470:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829984.776482:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a940 x1874672085953536/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829984.776490:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829984.776492:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787829984.776494:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829984.776497:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3420 00000100:00080000:3.0:1787829984.776500:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829984.776502:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829984.776506:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829984.776507:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085953280:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787829984.776517:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b0c0 x1874672085953664/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829984.776524:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829984.776526:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787829984.776528:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829984.776531:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3420 00000100:00080000:3.0:1787829984.776533:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829984.776536:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787829984.776540:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085953408:192.168.203.141@tcp:8:kworker.0 00000008:00100000:3.0:1787829984.776541:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:3.0:1787829984.776559:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499e00 x1874672085953792/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829984.776569:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787829984.776573:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085953536:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1787829984.776575:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787829984.776579:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829984.776586:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3420 00000100:00080000:3.0:1787829984.776590:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829984.776594:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787829984.776600:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:0.0:1787829984.776601:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085953664:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787829984.776614:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a580 x1874672085953920/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829984.776645:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085953792:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829984.776670:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085953920:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787829984.903938:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829984.903958:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829984.903997:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829984.904050:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829985.927929:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829985.927929:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787829985.927999:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829985.928129:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829986.951944:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829986.951957:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:0.0:1787829986.952464:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829986.952482:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000400:00000100:2.0:1787829987.848786:0:2399:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.141:988 00000400:00000100:2.0:1787829987.852577:0:2399:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.141@tcp at host 192.168.203.141 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1787829987.852601:0:2399:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:2.0:1787829987.852667:0:2399:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000800:00000100:2.0:1787829987.852680:0:2399:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:2.0:1787829987.852688:0:2399:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:2.0:1787829987.852696:0:2399:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:2.0:1787829987.852704:0:2399:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:2.0:1787829987.852711:0:2399:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:2.0:1787829987.852718:0:2399:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:2.0:1787829987.852726:0:2399:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:0.0:1787829987.852953:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829984/real 1787829987] req@ffff9e35c6499e00 x1874672085953792/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830009 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829987.852978:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499e00 x1874672085953792/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830009 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829987.852991:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829987.852996:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829987.853000:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085953792:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829987.853007:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829984/real 1787829987] req@ffff9e35c649a580 x1874672085953920/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830009 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829987.853017:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a580 x1874672085953920/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830009 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829987.853024:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829987.853028:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829987.853030:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085953920:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829987.853037:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829984/real 1787829987] req@ffff9e35c6499680 x1874672085953408/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830009 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829987.853047:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499680 x1874672085953408/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830009 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829987.853054:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829987.853058:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829987.853060:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085953408:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829987.853257:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829984/real 1787829987] req@ffff9e35c649a940 x1874672085953536/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830009 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829987.853273:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a940 x1874672085953536/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830009 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829987.853281:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829987.853284:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829987.853288:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085953536:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829987.853294:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829984/real 1787829987] req@ffff9e35c649b0c0 x1874672085953664/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830009 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829987.853304:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b0c0 x1874672085953664/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830009 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829987.853310:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829987.853314:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829987.853316:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085953664:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829987.853322:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829984/real 1787829987] req@ffff9e35c6498b40 x1874672085953024/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830009 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829987.853332:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498b40 x1874672085953024/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830009 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829987.853339:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829987.853342:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829987.853345:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085953024:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829987.853362:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829984/real 1787829987] req@ffff9e35c649bc00 x1874672085953152/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830009 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829987.853373:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649bc00 x1874672085953152/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830009 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829987.853380:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829987.853383:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829987.853386:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085953152:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829987.853392:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829984/real 1787829987] req@ffff9e35c649b840 x1874672085953280/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830009 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829987.853403:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b840 x1874672085953280/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830009 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829987.853410:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829987.853413:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829987.853416:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085953280:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829987.853422:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829984/real 1787829987] req@ffff9e35c649a1c0 x1874672085952896/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830009 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829987.853433:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a1c0 x1874672085952896/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830009 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829987.853439:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829987.853442:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829987.853445:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085952896:192.168.203.141@tcp:250:kworker.0 00010000:00010000:3.0:1787829987.975956:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829987.976000:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829987.976012:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829987.977202:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829988.999961:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829989.000077:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787829989.000327:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829989.000437:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00080000:2.0:1787829989.896018:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829989.896032:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829989.896037:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829989.896055:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3425 00000100:00080000:2.0:1787829989.896069:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829989.896076:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787829989.896120:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c3c0 x1874672085954048/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829989.896154:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829989.896158:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829989.896160:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829989.896165:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3425 00000100:00080000:2.0:1787829989.896169:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829989.896174:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829989.896187:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829989.896202:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f840 x1874672085954176/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829989.896212:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829989.896216:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829989.896218:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829989.896224:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3425 00000100:00100000:0.0:1787829989.896225:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085954048:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1787829989.896228:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829989.896235:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829989.896242:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829989.896256:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f0c0 x1874672085954304/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829989.896265:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829989.896268:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829989.896270:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829989.896275:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3425 00000100:00080000:2.0:1787829989.896280:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829989.896284:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829989.896289:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:2.0:1787829989.896302:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cf00 x1874672085954432/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829989.896311:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1787829989.896312:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1787829989.896317:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829989.896319:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829989.896325:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3425 00000100:00080000:2.0:1787829989.896330:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829989.896334:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829989.896355:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:2.0:1787829989.896368:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e580 x1874672085954560/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829989.896377:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829989.896380:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829989.896383:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787829989.896387:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829989/real 1787829989] req@ffff9e35c498c3c0 x1874672085954048/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830014 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829989.896391:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3425 00000100:00080000:2.0:1787829989.896397:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829989.896404:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829989.896409:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829989.896417:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c3c0 x1874672085954048/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830014 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829989.896426:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cb40 x1874672085954688/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829989.896436:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829989.896438:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829989.896443:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787829989.896446:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:2.0:1787829989.896447:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829989.896455:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3425 00000100:00100000:0.0:1787829989.896457:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085954048:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1787829989.896459:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829989.896467:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829989.896472:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829989.896482:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498d680 x1874672085954816/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829989.896489:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829989.896492:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829989.896495:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787829989.896496:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085954176:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1787829989.896502:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3425 00000100:00080000:2.0:1787829989.896507:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829989.896512:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787829989.896517:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:2.0:1787829989.896532:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e940 x1874672085954944/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829989.896542:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1787829989.896543:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1787829989.896546:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829989.896549:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829989.896555:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3425 00000100:00080000:2.0:1787829989.896559:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829989.896566:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787829989.896567:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829989/real 1787829989] req@ffff9e35c498f840 x1874672085954176/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830014 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1787829989.896571:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:2.0:1787829989.896586:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498de00 x1874672085955072/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829989.896589:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f840 x1874672085954176/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830014 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829989.896614:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829989.896618:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829989.896622:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085954176:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829989.896629:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085954304:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829989.896653:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829989.896664:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829989/real 1787829989] req@ffff9e35c498f0c0 x1874672085954304/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830014 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829989.896676:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f0c0 x1874672085954304/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830014 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829989.896684:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829989.896688:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829989.896692:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085954304:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829989.896698:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085954432:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829989.896716:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829989.896727:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829989/real 1787829989] req@ffff9e35c498cf00 x1874672085954432/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830014 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829989.896740:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498cf00 x1874672085954432/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830014 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829989.896748:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829989.896751:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829989.896755:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085954432:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829989.896760:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085954560:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829989.896779:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829989.896789:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829989/real 1787829989] req@ffff9e35c498e580 x1874672085954560/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830014 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829989.896836:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e580 x1874672085954560/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830014 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829989.896847:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829989.896850:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829989.896853:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085954560:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829989.896859:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085954688:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829989.896888:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829989.896898:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829989/real 1787829989] req@ffff9e35c498cb40 x1874672085954688/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830014 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829989.896910:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498cb40 x1874672085954688/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830014 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829989.896919:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829989.896922:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829989.896926:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085954688:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829989.896932:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085954816:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829989.896951:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829989.896961:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829989/real 1787829989] req@ffff9e35c498d680 x1874672085954816/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830014 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829989.896973:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498d680 x1874672085954816/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830014 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829989.896980:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829989.896983:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829989.896986:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085954816:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829989.897013:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085954944:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829989.897031:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829989.897042:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829989/real 1787829989] req@ffff9e35c498e940 x1874672085954944/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830014 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829989.897053:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e940 x1874672085954944/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830014 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829989.897061:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829989.897065:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829989.897068:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085954944:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829989.897074:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085955072:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829989.897090:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829989.897102:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829989/real 1787829989] req@ffff9e35c498de00 x1874672085955072/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830014 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829989.897116:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498de00 x1874672085955072/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830014 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829989.897124:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829989.897127:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829989.897130:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085955072:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829990.023913:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829990.023936:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829990.023946:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829990.023982:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829991.047974:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829991.048005:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829991.048570:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829991.048879:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829992.071945:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829992.072004:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:1.0:1787829992.072019:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829992.072035:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829993.096479:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829993.096535:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829993.096714:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829993.096821:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787829994.119952:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829994.119955:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829994.120019:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829994.120068:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00080000:1.0:1787829995.015972:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829995.015986:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829995.015991:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829995.017205:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3430 00000100:00080000:1.0:1787829995.017220:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829995.017229:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787829995.017270:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6d00 x1874672085955200/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829995.017294:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829995.017316:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829995.017319:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829995.017323:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3430 00000100:00080000:1.0:1787829995.017327:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829995.017331:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829995.017342:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829995.017356:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4780 x1874672085955328/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829995.017364:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085955200:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787829995.017369:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829995.017373:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829995.017374:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829995.017377:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3430 00000100:00080000:1.0:1787829995.017380:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829995.017383:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829995.017387:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829995.017395:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6580 x1874672085955456/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829995.017404:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829995.017406:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829995.017408:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829995.017411:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3430 00000100:00080000:1.0:1787829995.017414:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829995.017417:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829995.017422:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:1.0:1787829995.017430:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7480 x1874672085955584/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829995.017438:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829995.017441:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829995.017443:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829995.017447:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3430 00000100:00080000:1.0:1787829995.017450:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829995.017454:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829995.017458:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:1.0:1787829995.017468:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7840 x1874672085955712/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829995.017476:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829995.017478:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829995.017480:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829995.017484:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3430 00000100:00080000:1.0:1787829995.017487:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829995.017490:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829995.017494:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829995.017503:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4f00 x1874672085955840/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829995.017512:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829995.017514:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829995.017515:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829995.017517:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3430 00000100:00080000:1.0:1787829995.017519:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829995.017521:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829995.017524:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829995.017530:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f61c0 x1874672085955968/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829995.017535:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829995.017537:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829995.017538:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829995.017540:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3430 00000100:00080000:1.0:1787829995.017541:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829995.017543:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829995.017545:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:1.0:1787829995.017549:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6940 x1874672085956096/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829995.017557:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829995.017559:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829995.017560:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829995.017562:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3430 00000100:00080000:1.0:1787829995.017564:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829995.017565:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787829995.017567:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:1.0:1787829995.017572:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4000 x1874672085956224/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829995.018647:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829995.018697:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829995/real 1787829995] req@ffff9e35da6f6d00 x1874672085955200/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830020 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829995.018715:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6d00 x1874672085955200/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830020 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829995.018725:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829995.018729:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829995.018732:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085955200:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1787829995.018759:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085955328:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829995.018812:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829995.018825:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829995/real 1787829995] req@ffff9e35da6f4780 x1874672085955328/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830020 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829995.018841:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4780 x1874672085955328/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830020 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829995.018849:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829995.018852:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829995.018855:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085955328:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829995.018861:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085955456:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829995.018879:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829995.018888:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829995/real 1787829995] req@ffff9e35da6f6580 x1874672085955456/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830020 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829995.018898:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6580 x1874672085955456/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830020 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829995.018905:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829995.018908:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829995.018911:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085955456:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829995.018916:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085955584:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829995.018932:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829995.018942:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829995/real 1787829995] req@ffff9e35da6f7480 x1874672085955584/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830020 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829995.018953:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7480 x1874672085955584/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830020 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829995.018965:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829995.018968:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829995.018971:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085955584:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829995.018978:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085955712:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829995.018995:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829995.019005:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829995/real 1787829995] req@ffff9e35da6f7840 x1874672085955712/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830020 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829995.019015:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7840 x1874672085955712/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830020 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829995.019020:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829995.019022:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829995.019024:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085955712:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829995.019028:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085955840:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829995.019040:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829995.019046:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829995/real 1787829995] req@ffff9e35da6f4f00 x1874672085955840/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830020 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829995.019052:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4f00 x1874672085955840/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830020 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829995.019056:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829995.019058:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829995.019060:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085955840:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829995.019064:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085955968:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787829995.019073:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829995.019077:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829995/real 1787829995] req@ffff9e35da6f61c0 x1874672085955968/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830020 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829995.019083:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f61c0 x1874672085955968/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830020 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829995.019087:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829995.019089:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829995.019090:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085955968:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787829995.019093:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085956096:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829995.019102:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829995.019107:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829995/real 1787829995] req@ffff9e35da6f6940 x1874672085956096/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830020 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829995.019117:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6940 x1874672085956096/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830020 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829995.019124:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829995.019127:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829995.019130:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085956096:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829995.019136:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085956224:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787829995.019153:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787829995.019163:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829995/real 1787829995] req@ffff9e35da6f4000 x1874672085956224/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830020 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829995.019173:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4000 x1874672085956224/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830020 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829995.019181:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829995.019184:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787829995.019187:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085956224:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787829995.143928:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787829995.143930:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787829995.143931:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829995.143987:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829996.167883:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787829996.167934:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787829996.167941:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787829996.168002:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829997.191901:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787829997.191901:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787829997.191918:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787829997.191947:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829998.215919:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829998.215972:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829998.215980:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:3.0:1787829998.215992:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:3.0:1787829999.239915:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787829999.239916:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787829999.239928:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787829999.239979:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787830000.135934:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830000.135945:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787830000.135949:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830000.135965:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3435 00000100:00080000:1.0:1787830000.135976:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830000.135983:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787830000.136015:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f43c0 x1874672085956352/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830000.136035:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830000.136038:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787830000.136040:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830000.136042:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3435 00000100:00080000:1.0:1787830000.136044:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830000.136047:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830000.136059:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830000.136068:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f70c0 x1874672085956480/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830000.136072:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085956352:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787830000.136080:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830000.136084:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787830000.136085:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830000.136088:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3435 00000100:00080000:1.0:1787830000.136091:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830000.136093:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830000.136098:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830000.136111:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4b40 x1874672085956608/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830000.136121:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830000.136125:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787830000.136127:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1787830000.136132:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787830000.136136:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3435 00000100:00080000:1.0:1787830000.136142:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830000.136147:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830000.136153:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:1.0:1787830000.136166:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5e00 x1874672085956736/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830000.136177:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787830000.136180:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830000/real 1787830000] req@ffff9e35da6f43c0 x1874672085956352/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830025 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830000.136182:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787830000.136187:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830000.136196:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3435 00000100:00100000:0.0:1787830000.136200:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f43c0 x1874672085956352/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830025 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830000.136200:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830000.136209:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:0.0:1787830000.136212:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830000.136217:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000008:00100000:1.0:1787830000.136217:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:0.0:1787830000.136220:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085956352:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1787830000.136236:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5a40 x1874672085956864/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830000.136241:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085956480:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787830000.136258:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830000.136262:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787830000.136274:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1787830000.136278:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787830000.136281:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3435 00000100:00080000:1.0:1787830000.136288:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830000.136292:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787830000.136296:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830000/real 1787830000] req@ffff9e35da6f70c0 x1874672085956480/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830025 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1787830000.136298:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830000.136316:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f70c0 x1874672085956480/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830025 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830000.136317:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f43c0 x1874672085956992/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830000.136329:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830000.136333:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830000.136337:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:1.0:1787830000.136338:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787830000.136342:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787830000.136346:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085956480:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787830000.136348:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3435 00000100:00080000:1.0:1787830000.136352:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787830000.136360:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085956608:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787830000.136360:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830000.136370:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830000.136386:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4000 x1874672085957120/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787830000.136393:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830000.136403:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830000/real 1787830000] req@ffff9e35da6f4b40 x1874672085956608/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830025 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830000.136411:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787830000.136417:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4b40 x1874672085956608/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830025 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830000.136417:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787830000.136422:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830000.136429:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830000.136433:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3435 00000100:00080000:0.0:1787830000.136436:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830000.136441:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085956608:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787830000.136443:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787830000.136447:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085956736:192.168.203.141@tcp:8:kworker.0 00000100:00080000:1.0:1787830000.136459:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:0.0:1787830000.136464:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:1.0:1787830000.136467:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:0.0:1787830000.136474:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830000/real 1787830000] req@ffff9e35da6f5e00 x1874672085956736/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830025 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830000.136483:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6940 x1874672085957248/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830000.136485:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5e00 x1874672085956736/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830025 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830000.136513:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830000.136515:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830000.136519:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787830000.136522:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:1.0:1787830000.136523:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787830000.136527:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085956736:192.168.203.141@tcp:8:kworker.0 00000100:00080000:1.0:1787830000.136537:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3435 00000100:00080000:1.0:1787830000.136542:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830000.136546:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787830000.136550:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085956864:192.168.203.141@tcp:8:kworker.0 00000008:00100000:1.0:1787830000.136553:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:1.0:1787830000.136569:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f61c0 x1874672085957376/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787830000.136577:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830000.136592:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830000/real 1787830000] req@ffff9e35da6f5a40 x1874672085956864/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830025 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830000.136602:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5a40 x1874672085956864/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830025 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830000.136607:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830000.136610:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830000.136613:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085956864:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830000.136617:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085956992:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830000.136628:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830000.137734:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830000/real 1787830000] req@ffff9e35da6f43c0 x1874672085956992/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830025 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830000.137746:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f43c0 x1874672085956992/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830025 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830000.137751:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830000.137753:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830000.137756:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085956992:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830000.137760:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085957120:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830000.137787:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830000.137812:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830000/real 1787830000] req@ffff9e35da6f4000 x1874672085957120/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830025 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830000.137825:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4000 x1874672085957120/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830025 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830000.137834:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830000.137837:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830000.137840:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085957120:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830000.137847:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085957248:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830000.137867:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830000.137876:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830000/real 1787830000] req@ffff9e35da6f6940 x1874672085957248/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830025 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830000.137883:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6940 x1874672085957248/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830025 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830000.137888:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830000.137890:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830000.137892:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085957248:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830000.137908:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085957376:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830000.137919:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830000.137925:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830000/real 1787830000] req@ffff9e35da6f61c0 x1874672085957376/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830025 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830000.137931:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f61c0 x1874672085957376/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830025 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830000.137936:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830000.137938:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830000.137940:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085957376:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830000.263933:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830000.263934:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787830000.263992:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787830000.264072:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787830001.287948:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787830001.287949:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787830001.287949:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:0.0:1787830001.288019:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830002.311918:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787830002.311918:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787830002.311979:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830002.312161:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787830003.336211:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787830003.336333:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787830003.336904:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787830003.336957:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787830004.359942:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787830004.360013:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787830004.360136:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787830004.360203:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787830005.255949:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830005.255961:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787830005.255965:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830005.255976:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3440 00000100:00080000:1.0:1787830005.255986:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830005.255994:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787830005.256030:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4f00 x1874672085957504/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830005.256116:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085957504:192.168.203.141@tcp:250:kworker.0 00000800:00000100:0.0:1787830005.256221:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787830005.256279:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830005.256284:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787830005.256286:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830005.256291:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3440 00000100:00080000:1.0:1787830005.256295:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830005.256299:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830005.256309:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830005.256323:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7840 x1874672085957632/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830005.256654:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830005/real 1787830005] req@ffff9e35da6f4f00 x1874672085957504/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830030 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830005.256672:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4f00 x1874672085957504/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830030 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830005.256682:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830005.256686:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830005.256690:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085957504:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1787830005.256731:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085957632:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830005.256755:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830005.256768:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830005/real 1787830005] req@ffff9e35da6f7840 x1874672085957632/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830030 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830005.256781:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7840 x1874672085957632/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830030 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830005.256789:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830005.256792:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830005.256796:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085957632:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787830005.256976:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830005.256981:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787830005.256983:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830005.256988:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3440 00000100:00080000:1.0:1787830005.256993:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830005.256997:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830005.257003:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830005.257017:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7480 x1874672085957760/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830005.257027:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830005.257030:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787830005.257032:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830005.257035:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3440 00000100:00080000:1.0:1787830005.257039:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830005.257042:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830005.257046:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:1.0:1787830005.257056:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6580 x1874672085957888/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830005.257065:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830005.257068:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787830005.257071:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830005.257074:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3440 00000100:00080000:1.0:1787830005.257077:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830005.257080:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830005.257085:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:1.0:1787830005.257094:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4780 x1874672085958016/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830005.257103:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830005.257106:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787830005.257107:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830005.257111:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3440 00000100:00080000:1.0:1787830005.257114:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830005.257118:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830005.257123:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830005.257134:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6d00 x1874672085958144/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830005.257143:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830005.257146:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787830005.257147:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830005.257150:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3440 00000100:00080000:1.0:1787830005.257153:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830005.257157:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830005.257161:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830005.257171:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4f00 x1874672085958272/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830005.257180:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830005.257183:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787830005.257185:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830005.257275:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3440 00000100:00080000:1.0:1787830005.257280:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830005.257284:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830005.257289:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:1.0:1787830005.257301:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f61c0 x1874672085958400/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830005.257311:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830005.257314:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787830005.257315:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830005.257319:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3440 00000100:00080000:1.0:1787830005.257322:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830005.257325:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830005.257330:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:1.0:1787830005.257339:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6940 x1874672085958528/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830005.257407:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085957760:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830005.257477:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830005.257495:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830005/real 1787830005] req@ffff9e35da6f7480 x1874672085957760/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830030 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830005.257503:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7480 x1874672085957760/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830030 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830005.257518:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830005.257525:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830005.257528:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085957760:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830005.257532:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085957888:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830005.257550:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830005.257557:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830005/real 1787830005] req@ffff9e35da6f6580 x1874672085957888/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830030 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830005.257563:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6580 x1874672085957888/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830030 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830005.257588:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830005.257590:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830005.257610:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085957888:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830005.257614:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085958016:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830005.257633:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830005.257640:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830005/real 1787830005] req@ffff9e35da6f4780 x1874672085958016/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830030 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830005.257647:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4780 x1874672085958016/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830030 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830005.257670:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830005.257674:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830005.257677:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085958016:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830005.257682:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085958144:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830005.257701:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830005.257712:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830005/real 1787830005] req@ffff9e35da6f6d00 x1874672085958144/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830030 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830005.257724:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6d00 x1874672085958144/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830030 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830005.257731:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830005.257734:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830005.257737:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085958144:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830005.257743:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085958272:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830005.257762:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830005.257858:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830005/real 1787830005] req@ffff9e35da6f4f00 x1874672085958272/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830030 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830005.257873:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4f00 x1874672085958272/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830030 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830005.257881:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830005.257884:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830005.257887:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085958272:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830005.257893:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085958400:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830005.257914:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830005.257926:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830005/real 1787830005] req@ffff9e35da6f61c0 x1874672085958400/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830030 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830005.257937:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f61c0 x1874672085958400/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830030 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830005.257945:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830005.257948:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830005.257951:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085958400:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830005.257956:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085958528:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830005.257975:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830005.257986:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830005/real 1787830005] req@ffff9e35da6f6940 x1874672085958528/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830030 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830005.257997:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6940 x1874672085958528/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830030 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830005.258004:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830005.258007:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830005.258010:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085958528:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787830005.383937:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787830005.383938:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787830005.383970:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787830005.384005:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787830006.407937:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787830006.407952:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830006.408016:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:3.0:1787830006.408032:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787830007.431935:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:3.0:1787830007.431935:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787830007.432012:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787830007.434061:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787830008.455949:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:3.0:1787830008.455949:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:0.0:1787830008.456020:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787830008.456078:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787830009.479936:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787830009.479974:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787830009.480082:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:3.0:1787830009.481247:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787830010.376036:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830010.376047:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787830010.376051:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830010.376062:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3445 00000100:00080000:3.0:1787830010.376072:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830010.376078:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787830010.376112:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499a40 x1874672085958656/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830010.376146:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830010.376150:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787830010.376151:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830010.376155:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3445 00000100:00080000:3.0:1787830010.376158:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830010.376162:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830010.376171:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830010.376182:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498000 x1874672085958784/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830010.376192:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830010.376205:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787830010.376207:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830010.376213:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3445 00000100:00100000:0.0:1787830010.376215:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085958656:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1787830010.376216:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830010.376223:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830010.376228:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830010.376239:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649ad00 x1874672085958912/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830010.376250:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830010.376253:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787830010.376255:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830010.376259:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3445 00000100:00080000:3.0:1787830010.376262:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830010.376266:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830010.376270:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000800:00000100:0.0:1787830010.376278:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787830010.376282:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c64983c0 x1874672085959040/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830010.376291:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830010.376295:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787830010.376297:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830010.376301:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3445 00000100:00080000:3.0:1787830010.376304:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830010.376307:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830010.376311:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:0.0:1787830010.376322:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830010/real 1787830010] req@ffff9e35c6499a40 x1874672085958656/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830035 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787830010.376324:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c64992c0 x1874672085959168/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830010.376349:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830010.376353:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:0.0:1787830010.376356:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499a40 x1874672085958656/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830035 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830010.376356:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830010.376364:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3445 00000100:00080000:3.0:1787830010.376370:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830010.376375:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787830010.376376:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830010.376383:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1787830010.376384:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830010.376391:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085958656:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1787830010.376396:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b480 x1874672085959296/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830010.376402:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085958784:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1787830010.376413:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830010.376416:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787830010.376418:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830010.376422:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3445 00000100:00080000:3.0:1787830010.376426:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1787830010.376430:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1787830010.376432:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830010.376438:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830010.376445:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830010/real 1787830010] req@ffff9e35c6498000 x1874672085958784/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830035 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787830010.376450:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498780 x1874672085959424/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830010.376469:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787830010.376470:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498000 x1874672085958784/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830035 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830010.376473:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787830010.376477:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830010.376486:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3445 00000100:00080000:0.0:1787830010.376488:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787830010.376489:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830010.376497:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:3.0:1787830010.376498:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787830010.376504:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085958784:192.168.203.141@tcp:38:kworker.0 00000008:00100000:3.0:1787830010.376504:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:3.0:1787830010.376519:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498f00 x1874672085959552/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830010.376528:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830010.376531:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:0.0:1787830010.376534:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085958912:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1787830010.376534:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830010.376549:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3445 00000100:00080000:3.0:1787830010.376553:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1787830010.376558:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1787830010.376561:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830010.376567:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:0.0:1787830010.376572:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830010/real 1787830010] req@ffff9e35c649ad00 x1874672085958912/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830035 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787830010.376579:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498000 x1874672085959680/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830010.376590:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649ad00 x1874672085958912/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830035 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830010.376600:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830010.376604:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830010.376607:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085958912:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830010.376614:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c64983c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085959040:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830010.376629:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830010.376638:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830010/real 1787830010] req@ffff9e35c64983c0 x1874672085959040/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830035 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830010.376654:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c64983c0 x1874672085959040/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830035 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830010.376662:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830010.376665:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830010.376668:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c64983c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085959040:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830010.376673:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c64992c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085959168:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830010.376687:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830010.376695:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830010/real 1787830010] req@ffff9e35c64992c0 x1874672085959168/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830035 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830010.376705:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c64992c0 x1874672085959168/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830035 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830010.376713:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830010.376716:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830010.376719:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c64992c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085959168:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830010.376724:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085959296:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830010.376737:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830010.376744:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830010/real 1787830010] req@ffff9e35c649b480 x1874672085959296/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830035 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830010.376754:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b480 x1874672085959296/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830035 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830010.376767:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830010.376770:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830010.376773:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085959296:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830010.376778:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085959424:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830010.376792:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830010.376862:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830010/real 1787830010] req@ffff9e35c6498780 x1874672085959424/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830035 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830010.376874:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498780 x1874672085959424/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830035 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830010.376882:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830010.376886:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830010.376889:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085959424:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830010.376896:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085959552:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830010.376913:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830010.376929:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830010/real 1787830010] req@ffff9e35c6498f00 x1874672085959552/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830035 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830010.376939:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498f00 x1874672085959552/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830035 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830010.376947:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830010.376951:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830010.376953:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085959552:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830010.376971:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085959680:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830010.376986:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830010.376994:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830010/real 1787830010] req@ffff9e35c6498000 x1874672085959680/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830035 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830010.377005:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498000 x1874672085959680/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830035 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830010.377012:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830010.377015:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830010.377018:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085959680:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787830010.503939:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787830010.503981:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787830010.504037:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:1.0:1787830010.504037:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830011.527866:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787830011.527870:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787830011.528089:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787830011.528213:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787830012.551996:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787830012.553327:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830012.553378:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:3.0:1787830012.553552:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787830013.575964:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787830013.575970:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787830013.576204:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830013.576406:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787830014.599975:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787830014.601448:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787830014.601515:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830014.601575:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00080000:3.0:1787830015.495941:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830015.495953:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787830015.495958:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830015.495968:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3450 00000100:00080000:3.0:1787830015.495980:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830015.495987:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787830015.496025:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499a40 x1874672085959808/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830015.496106:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085959808:192.168.203.141@tcp:250:kworker.0 00000800:00000100:0.0:1787830015.496174:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1787830015.496204:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830015.496208:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787830015.496211:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830015.496216:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3450 00000100:00080000:3.0:1787830015.496220:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830015.496224:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830015.496235:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830015.496249:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a1c0 x1874672085959936/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830015.496334:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830015/real 1787830015] req@ffff9e35c6499a40 x1874672085959808/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830040 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830015.496351:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499a40 x1874672085959808/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830040 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830015.496361:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830015.496365:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830015.496369:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085959808:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1787830015.496390:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085959936:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830015.496413:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830015.496425:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830015/real 1787830015] req@ffff9e35c649a1c0 x1874672085959936/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830040 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830015.496437:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a1c0 x1874672085959936/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830040 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830015.496445:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830015.496448:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830015.496451:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085959936:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1787830015.496547:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830015.496551:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787830015.496553:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830015.496557:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3450 00000100:00080000:3.0:1787830015.496561:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830015.496565:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830015.496571:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830015.496583:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b840 x1874672085960064/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830015.496625:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085960064:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830015.496646:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1787830015.496668:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830015.496672:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787830015.496674:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830015.496679:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3450 00000100:00080000:3.0:1787830015.496683:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830015.496686:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830015.496691:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:3.0:1787830015.496702:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649bc00 x1874672085960192/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830015.496740:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830015/real 1787830015] req@ffff9e35c649b840 x1874672085960064/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830040 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830015.496753:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b840 x1874672085960064/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830040 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830015.496761:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830015.496764:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830015.496767:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085960064:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830015.496778:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085960192:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830015.496835:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830015.496885:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830015/real 1787830015] req@ffff9e35c649bc00 x1874672085960192/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830040 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830015.496899:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649bc00 x1874672085960192/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830040 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830015.496908:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830015.496912:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830015.496916:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085960192:192.168.203.141@tcp:8:kworker.0 00000100:00080000:3.0:1787830015.497066:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830015.497070:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787830015.497072:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830015.497076:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3450 00000100:00080000:3.0:1787830015.497081:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830015.497085:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830015.497090:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:3.0:1787830015.497104:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498b40 x1874672085960320/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830015.497145:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085960320:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830015.497179:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1787830015.497201:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830015.497204:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787830015.497206:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830015.497211:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3450 00000100:00080000:3.0:1787830015.497215:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830015.497219:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830015.497224:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830015.497235:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b0c0 x1874672085960448/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830015.497271:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830015/real 1787830015] req@ffff9e35c6498b40 x1874672085960320/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830040 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830015.497284:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498b40 x1874672085960320/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830040 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830015.497292:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830015.497295:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830015.497298:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085960320:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830015.497310:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085960448:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830015.497328:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830015.497338:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830015/real 1787830015] req@ffff9e35c649b0c0 x1874672085960448/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830040 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830015.497368:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b0c0 x1874672085960448/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830040 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830015.497377:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830015.497380:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830015.497383:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085960448:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1787830015.497656:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830015.497660:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787830015.497662:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830015.497667:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3450 00000100:00080000:3.0:1787830015.497671:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830015.497675:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830015.497681:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830015.497693:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a940 x1874672085960576/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830015.497732:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085960576:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830015.497755:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1787830015.497775:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830015.497778:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787830015.497780:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830015.497784:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3450 00000100:00080000:3.0:1787830015.497788:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830015.497792:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830015.497815:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:3.0:1787830015.497827:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499680 x1874672085960704/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830015.497877:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830015/real 1787830015] req@ffff9e35c649a940 x1874672085960576/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830040 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830015.497890:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a940 x1874672085960576/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830040 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830015.497897:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830015.497900:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830015.497903:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085960576:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830015.497916:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085960704:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830015.497936:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830015.497946:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830015/real 1787830015] req@ffff9e35c6499680 x1874672085960704/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830040 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830015.497957:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499680 x1874672085960704/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830040 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830015.497970:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830015.497973:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830015.497976:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085960704:192.168.203.141@tcp:8:kworker.0 00000100:00080000:3.0:1787830015.498063:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830015.498067:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787830015.498069:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830015.498073:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3450 00000100:00080000:3.0:1787830015.498077:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830015.498081:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830015.498086:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:3.0:1787830015.498097:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a580 x1874672085960832/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830015.498138:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085960832:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830015.498169:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830015.498243:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830015/real 1787830015] req@ffff9e35c649a580 x1874672085960832/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830040 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830015.498256:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a580 x1874672085960832/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830040 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830015.498265:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830015.498268:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830015.498270:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085960832:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830015.623888:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787830015.623924:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787830015.625769:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787830015.625775:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830016.647957:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:3.0:1787830016.647991:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787830016.647994:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830016.648116:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:3.0:1787830017.672038:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787830017.672051:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787830017.672096:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830017.672153:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787830018.695913:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787830018.696100:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787830018.696370:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787830018.696524:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787830019.719928:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:0.0:1787830019.720052:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787830019.720080:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:3.0:1787830019.720087:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787830020.616199:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830020.616293:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787830020.616298:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830020.616309:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3455 00000100:00080000:0.0:1787830020.616321:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830020.616330:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787830020.616373:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a1c0 x1874672085960960/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830020.616406:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830020.616410:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787830020.616412:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830020.616416:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3455 00000100:00080000:0.0:1787830020.616420:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830020.616424:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830020.616436:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830020.616457:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718b40 x1874672085961088/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787830020.616459:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085960960:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1787830020.616481:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830020.616485:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787830020.616487:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830020.616492:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3455 00000100:00080000:0.0:1787830020.616496:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830020.616500:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830020.616505:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830020.616517:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a580 x1874672085961216/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830020.616527:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830020.616531:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787830020.616533:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1787830020.616541:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787830020.616543:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3455 00000100:00080000:0.0:1787830020.616551:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830020.616556:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830020.616562:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:0.0:1787830020.616578:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718000 x1874672085961344/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787830020.616589:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830020/real 1787830020] req@ffff9e35da71a1c0 x1874672085960960/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830045 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830020.616592:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830020.616611:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787830020.616622:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830020.616633:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3455 00000100:00100000:2.0:1787830020.616634:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a1c0 x1874672085960960/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830045 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830020.616641:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830020.616656:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:2.0:1787830020.616660:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:0.0:1787830020.616664:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00080000:2.0:1787830020.616673:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787830020.616680:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085960960:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1787830020.616684:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7183c0 x1874672085961472/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830020.616699:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830020.616703:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787830020.616705:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830020.616712:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3455 00000100:00100000:2.0:1787830020.616716:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085961088:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787830020.616719:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830020.616729:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830020.616736:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1787830020.616750:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830020.616768:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da719a40 x1874672085961600/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787830020.616769:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830020/real 1787830020] req@ffff9e35da718b40 x1874672085961088/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830045 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830020.616839:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718b40 x1874672085961088/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830045 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830020.616849:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830020.616861:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787830020.616872:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085961088:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787830020.616893:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085961216:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787830020.616897:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830020.616902:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787830020.616906:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830020.616912:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3455 00000100:00080000:0.0:1787830020.616917:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1787830020.616918:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787830020.616928:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830020.616935:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830020.616936:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830020/real 1787830020] req@ffff9e35da71a580 x1874672085961216/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830045 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830020.616959:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718780 x1874672085961728/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787830020.616961:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672085961216/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830045 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830020.617015:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830020.617018:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830020.617023:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:0.0:1787830020.617026:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:2.0:1787830020.617032:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085961216:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787830020.617033:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830020.617050:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3455 00000100:00100000:2.0:1787830020.617057:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085961344:192.168.203.141@tcp:8:kworker.0 00000100:00080000:0.0:1787830020.617060:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830020.617073:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830020.617081:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000800:00000100:2.0:1787830020.617090:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830020.617098:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a940 x1874672085961856/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787830020.617111:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830020/real 1787830020] req@ffff9e35da718000 x1874672085961344/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830045 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830020.617113:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830020.617136:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787830020.617138:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830020.617149:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3455 00000100:00080000:0.0:1787830020.617155:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787830020.617157:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718000 x1874672085961344/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830045 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830020.617162:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830020.617174:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00080000:2.0:1787830020.617192:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830020.617196:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787830020.617198:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085961344:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787830020.617204:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085961472:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787830020.617223:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787830020.617231:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830020/real 1787830020] req@ffff9e35da7183c0 x1874672085961472/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830045 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830020.617241:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672085961472/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830045 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830020.617249:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830020.617253:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787830020.617256:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085961472:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787830020.617273:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085961600:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830020.617292:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718f00 x1874672085961984/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787830020.617299:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787830020.617316:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830020/real 1787830020] req@ffff9e35da719a40 x1874672085961600/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830045 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830020.617329:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672085961600/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830045 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830020.617337:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830020.617341:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787830020.617345:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085961600:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787830020.617351:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085961728:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787830020.617365:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787830020.617374:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830020/real 1787830020] req@ffff9e35da718780 x1874672085961728/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830045 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830020.617384:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718780 x1874672085961728/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830045 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830020.617391:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830020.617394:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787830020.617397:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085961728:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787830020.617402:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085961856:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787830020.617414:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787830020.617421:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830020/real 1787830020] req@ffff9e35da71a940 x1874672085961856/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830045 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830020.617431:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672085961856/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830045 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830020.617439:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830020.617442:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787830020.617445:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085961856:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787830020.617458:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085961984:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787830020.617471:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787830020.617479:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830020/real 1787830020] req@ffff9e35da718f00 x1874672085961984/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830045 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830020.617489:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672085961984/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830045 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830020.617497:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830020.617499:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787830020.617502:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085961984:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787830020.743955:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:3.0:1787830020.743987:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787830020.744061:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787830020.744163:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787830021.768005:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830021.768075:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787830021.768141:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830021.768148:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787830022.791938:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787830022.792193:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:3.0:1787830022.792244:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787830022.792305:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787830023.816278:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787830023.816404:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787830023.816686:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830023.816781:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830024.839890:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787830024.840009:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830024.841035:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787830024.841472:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00080000:1.0:1787830025.736517:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830025.736528:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787830025.736532:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830025.736543:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3461 00000100:00080000:1.0:1787830025.736553:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830025.736560:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787830025.736595:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4000 x1874672085962112/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830025.736625:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830025.736628:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787830025.736630:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830025.736634:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3461 00000100:00080000:1.0:1787830025.736637:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830025.736641:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830025.736651:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830025.736662:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f43c0 x1874672085962240/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830025.736672:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830025.736674:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787830025.736676:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830025.736680:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3461 00000100:00080000:1.0:1787830025.736683:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830025.736686:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830025.736691:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830025.736702:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5a40 x1874672085962368/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830025.736711:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830025.736714:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787830025.736715:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830025.736719:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3461 00000100:00080000:1.0:1787830025.736721:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830025.736725:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830025.736729:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:1.0:1787830025.736737:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5e00 x1874672085962496/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830025.736746:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830025.736749:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787830025.736751:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830025.736754:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3461 00000100:00080000:1.0:1787830025.736758:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830025.736761:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830025.736765:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:1.0:1787830025.736774:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4b40 x1874672085962624/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830025.736782:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830025.736785:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787830025.736787:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830025.736790:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3461 00000100:00080000:1.0:1787830025.736793:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830025.736796:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830025.737214:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830025.737229:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f70c0 x1874672085962752/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830025.737239:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830025.737242:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787830025.737244:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830025.737247:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3461 00000100:00080000:1.0:1787830025.737251:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830025.737256:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830025.737260:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830025.737271:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6940 x1874672085962880/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830025.737279:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830025.737283:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787830025.737285:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830025.737288:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3461 00000100:00080000:1.0:1787830025.737291:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830025.737294:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830025.737298:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:1.0:1787830025.737307:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f61c0 x1874672085963008/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830025.737316:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830025.737319:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787830025.737320:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830025.737323:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3461 00000100:00080000:1.0:1787830025.737326:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830025.737329:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830025.737333:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:1.0:1787830025.737342:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4f00 x1874672085963136/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787830025.737425:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085962112:192.168.203.141@tcp:250:kworker.0 00000800:00000100:2.0:1787830025.737493:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787830025.737530:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830025/real 1787830025] req@ffff9e35da6f4000 x1874672085962112/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830050 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830025.737548:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4000 x1874672085962112/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830050 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830025.737557:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830025.737560:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787830025.737564:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085962112:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1787830025.737571:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085962240:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787830025.737595:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787830025.737608:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830025/real 1787830025] req@ffff9e35da6f43c0 x1874672085962240/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830050 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830025.737619:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f43c0 x1874672085962240/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830050 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830025.737628:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830025.737631:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787830025.737634:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085962240:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787830025.737639:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085962368:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787830025.737657:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787830025.737666:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830025/real 1787830025] req@ffff9e35da6f5a40 x1874672085962368/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830050 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830025.737676:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5a40 x1874672085962368/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830050 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830025.737684:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830025.737687:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787830025.737690:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085962368:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787830025.737696:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085962496:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787830025.737716:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787830025.737727:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830025/real 1787830025] req@ffff9e35da6f5e00 x1874672085962496/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830050 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830025.737744:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5e00 x1874672085962496/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830050 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830025.737752:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830025.737754:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787830025.737757:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085962496:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787830025.737762:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085962624:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787830025.737781:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787830025.737791:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830025/real 1787830025] req@ffff9e35da6f4b40 x1874672085962624/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830050 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830025.737838:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4b40 x1874672085962624/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830050 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830025.737848:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830025.737851:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787830025.737854:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085962624:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787830025.737859:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085962752:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787830025.737879:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787830025.737889:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830025/real 1787830025] req@ffff9e35da6f70c0 x1874672085962752/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830050 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830025.737900:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f70c0 x1874672085962752/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830050 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830025.737913:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830025.737916:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787830025.737919:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085962752:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787830025.737925:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085962880:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787830025.737942:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787830025.737952:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830025/real 1787830025] req@ffff9e35da6f6940 x1874672085962880/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830050 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830025.737963:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6940 x1874672085962880/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830050 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830025.737971:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830025.737973:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787830025.737976:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085962880:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787830025.737982:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085963008:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787830025.737999:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787830025.738010:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830025/real 1787830025] req@ffff9e35da6f61c0 x1874672085963008/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830050 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830025.738021:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f61c0 x1874672085963008/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830050 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830025.738029:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830025.738032:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787830025.738035:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085963008:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787830025.738040:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085963136:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787830025.738058:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787830025.738069:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830025/real 1787830025] req@ffff9e35da6f4f00 x1874672085963136/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830050 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830025.738080:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4f00 x1874672085963136/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830050 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830025.738098:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830025.738101:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787830025.738104:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085963136:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830025.863945:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787830025.863951:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:3.0:1787830025.864000:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787830025.864082:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830026.887918:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787830026.887918:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787830026.887952:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830026.888211:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787830027.911929:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787830027.911929:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787830027.912047:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830027.912096:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787830028.935956:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787830028.935973:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787830028.935998:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:1.0:1787830028.936021:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830029.959924:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787830029.959925:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787830029.959925:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787830029.960026:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787830030.857316:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830030.857327:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787830030.857332:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830030.857353:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3466 00000100:00080000:3.0:1787830030.857364:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830030.857372:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787830030.857407:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499e00 x1874672085963264/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830030.857438:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830030.857442:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787830030.857443:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830030.857447:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3466 00000100:00080000:3.0:1787830030.857451:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830030.857455:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830030.857464:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830030.857479:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a580 x1874672085963392/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830030.857488:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830030.857491:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787830030.857493:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830030.857497:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3466 00000100:00080000:3.0:1787830030.857500:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830030.857503:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830030.857508:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830030.857518:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499680 x1874672085963520/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830030.857531:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830030.857534:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787830030.857536:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830030.857539:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3466 00000100:00080000:3.0:1787830030.857542:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830030.857546:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830030.857551:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:3.0:1787830030.857561:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a940 x1874672085963648/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830030.857570:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830030.857573:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787830030.857575:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830030.857578:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3466 00000100:00080000:3.0:1787830030.857581:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830030.857584:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830030.857588:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:3.0:1787830030.857598:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b0c0 x1874672085963776/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830030.857607:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830030.857610:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787830030.857611:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830030.857614:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3466 00000100:00080000:3.0:1787830030.857617:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830030.857621:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830030.857625:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830030.857634:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498b40 x1874672085963904/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830030.857642:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830030.857645:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787830030.857647:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830030.857650:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3466 00000100:00080000:3.0:1787830030.857653:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830030.857656:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830030.857661:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830030.857670:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649bc00 x1874672085964032/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830030.857678:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830030.857681:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787830030.857683:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830030.857687:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3466 00000100:00080000:3.0:1787830030.857690:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830030.857693:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830030.857698:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:3.0:1787830030.857707:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b840 x1874672085964160/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830030.857715:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830030.857719:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787830030.857720:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830030.857723:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3466 00000100:00080000:3.0:1787830030.857726:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830030.857729:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830030.857733:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:3.0:1787830030.857744:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a1c0 x1874672085964288/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787830030.857886:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085963264:192.168.203.141@tcp:250:kworker.0 00000800:00000100:2.0:1787830030.857947:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787830030.857987:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830030/real 1787830030] req@ffff9e35c6499e00 x1874672085963264/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830055 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830030.858005:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499e00 x1874672085963264/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830055 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830030.858015:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830030.858020:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787830030.858024:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085963264:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1787830030.858032:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085963392:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787830030.858065:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787830030.858079:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830030/real 1787830030] req@ffff9e35c649a580 x1874672085963392/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830055 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830030.858091:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a580 x1874672085963392/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830055 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830030.858100:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830030.858103:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787830030.858106:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085963392:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787830030.858111:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085963520:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787830030.858132:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787830030.858142:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830030/real 1787830030] req@ffff9e35c6499680 x1874672085963520/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830055 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830030.858153:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499680 x1874672085963520/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830055 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830030.858160:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830030.858163:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787830030.858166:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085963520:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787830030.858171:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085963648:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787830030.858190:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787830030.858201:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830030/real 1787830030] req@ffff9e35c649a940 x1874672085963648/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830055 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830030.858212:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a940 x1874672085963648/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830055 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830030.858220:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830030.858223:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787830030.858225:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085963648:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787830030.858231:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085963776:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787830030.858249:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787830030.858259:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830030/real 1787830030] req@ffff9e35c649b0c0 x1874672085963776/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830055 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830030.858270:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b0c0 x1874672085963776/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830055 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830030.858278:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830030.858281:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787830030.858285:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085963776:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787830030.858290:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085963904:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787830030.858308:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787830030.858319:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830030/real 1787830030] req@ffff9e35c6498b40 x1874672085963904/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830055 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830030.858337:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498b40 x1874672085963904/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830055 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830030.858355:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830030.858359:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787830030.858362:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085963904:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787830030.858368:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085964032:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787830030.858388:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787830030.858399:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830030/real 1787830030] req@ffff9e35c649bc00 x1874672085964032/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830055 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830030.858410:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649bc00 x1874672085964032/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830055 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830030.858417:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830030.858420:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787830030.858423:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085964032:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787830030.858428:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085964160:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787830030.858445:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787830030.858455:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830030/real 1787830030] req@ffff9e35c649b840 x1874672085964160/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830055 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830030.858466:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b840 x1874672085964160/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830055 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830030.858480:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830030.858483:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787830030.858486:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085964160:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787830030.858492:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085964288:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787830030.858511:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787830030.858522:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830030/real 1787830030] req@ffff9e35c649a1c0 x1874672085964288/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830055 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830030.858534:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a1c0 x1874672085964288/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830055 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830030.858542:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830030.858545:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787830030.858548:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085964288:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830030.983924:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830030.983924:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787830030.984090:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787830030.984128:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787830032.007913:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787830032.007948:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830032.011449:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787830032.011459:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830033.031940:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787830033.032485:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830033.032564:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787830033.032871:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787830034.055989:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830034.056122:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830034.058149:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787830034.058248:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787830035.079918:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830035.079919:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787830035.079919:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787830035.079977:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787830035.976157:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830035.976172:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787830035.976176:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830035.976187:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3471 00000100:00080000:3.0:1787830035.976198:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830035.976314:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787830035.976345:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499a40 x1874672085964416/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830035.976373:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830035.976376:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787830035.976377:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830035.976380:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3471 00000100:00080000:3.0:1787830035.976384:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787830035.976414:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085964416:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1787830035.976450:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:2.0:1787830035.976463:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:3.0:1787830035.976466:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830035.976483:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498000 x1874672085964544/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787830035.976493:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830035/real 1787830035] req@ffff9e35c6499a40 x1874672085964416/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830060 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830035.976495:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830035.976505:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787830035.976509:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830035.976518:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3471 00000100:00080000:3.0:1787830035.976523:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787830035.976525:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499a40 x1874672085964416/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830060 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830035.976530:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:2.0:1787830035.976536:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:3.0:1787830035.976539:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1787830035.976541:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787830035.976546:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085964416:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1787830035.976558:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498f00 x1874672085964672/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787830035.976566:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085964544:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1787830035.976575:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830035.976580:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787830035.976583:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1787830035.976585:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1787830035.976591:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3471 00000100:00080000:3.0:1787830035.976596:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830035.976603:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830035.976609:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:2.0:1787830035.976614:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830035/real 1787830035] req@ffff9e35c6498000 x1874672085964544/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830060 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787830035.976625:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498780 x1874672085964800/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787830035.976632:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498000 x1874672085964544/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830060 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830035.976638:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830035.976647:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787830035.976648:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787830035.976649:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830035.976657:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:3.0:1787830035.976659:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3471 00000100:00100000:2.0:1787830035.976664:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085964544:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1787830035.976666:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830035.976674:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830035.976679:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:2.0:1787830035.976681:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085964672:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787830035.976697:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787830035.976704:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830035/real 1787830035] req@ffff9e35c6498f00 x1874672085964672/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830060 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830035.976712:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498f00 x1874672085964672/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830060 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830035.976721:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830035.976723:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787830035.976725:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085964672:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787830035.976728:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085964800:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787830035.976739:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787830035.976748:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830035/real 1787830035] req@ffff9e35c6498780 x1874672085964800/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830060 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830035.976759:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498780 x1874672085964800/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830060 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830035.976767:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830035.976770:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787830035.976773:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085964800:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787830035.976844:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b480 x1874672085964928/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830035.976863:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830035.976867:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787830035.976869:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830035.976874:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3471 00000100:00080000:3.0:1787830035.976879:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830035.976884:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830035.976890:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830035.976902:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085964928:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787830035.976905:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c64992c0 x1874672085965056/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787830035.976925:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787830035.976937:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830035/real 1787830035] req@ffff9e35c649b480 x1874672085964928/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830060 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830035.976946:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b480 x1874672085964928/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830060 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830035.976953:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830035.976955:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787830035.976959:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085964928:192.168.203.141@tcp:8:kworker.0 00000100:00080000:3.0:1787830035.976961:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830035.976966:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:2.0:1787830035.976970:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c64992c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085965056:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1787830035.976970:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830035.976980:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3471 00000100:00080000:3.0:1787830035.976986:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1787830035.976988:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1787830035.976993:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787830035.976998:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830035/real 1787830035] req@ffff9e35c64992c0 x1874672085965056/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830060 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1787830035.977002:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830035.977028:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c64992c0 x1874672085965056/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830060 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787830035.977031:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c64983c0 x1874672085965184/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830035.977050:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787830035.977055:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830035.977056:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:3.0:1787830035.977063:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:2.0:1787830035.977064:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c64992c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085965056:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1787830035.977066:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830035.977074:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3471 00000100:00080000:3.0:1787830035.977079:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787830035.977082:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c64983c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085965184:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1787830035.977086:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830035.977093:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000800:00000100:2.0:1787830035.977106:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787830035.977120:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649ad00 x1874672085965312/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787830035.977121:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830035/real 1787830035] req@ffff9e35c64983c0 x1874672085965184/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830060 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830035.977141:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830035.977145:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787830035.977148:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830035.977155:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3471 00000100:00100000:2.0:1787830035.977159:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c64983c0 x1874672085965184/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830060 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830035.977160:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830035.977168:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:2.0:1787830035.977170:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000008:00100000:3.0:1787830035.977174:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00080000:2.0:1787830035.977176:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787830035.977182:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c64983c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085965184:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787830035.977189:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c64992c0 x1874672085965440/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787830035.977196:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085965312:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787830035.977229:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787830035.977237:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830035/real 1787830035] req@ffff9e35c649ad00 x1874672085965312/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830060 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830035.977244:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649ad00 x1874672085965312/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830060 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830035.977249:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830035.977251:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787830035.977253:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085965312:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787830035.977260:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c64992c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085965440:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787830035.977273:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787830035.977294:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830035/real 1787830035] req@ffff9e35c64992c0 x1874672085965440/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830060 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830035.977315:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c64992c0 x1874672085965440/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830060 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830035.977323:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830035.977325:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787830035.977328:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c64992c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085965440:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830036.103926:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787830036.103950:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787830036.104044:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787830036.104066:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787830037.128061:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830037.128090:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787830037.128114:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830037.128132:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787830038.151938:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:0.0:1787830038.151996:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787830038.152260:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830038.152506:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787830039.175945:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830039.177277:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787830039.177550:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787830039.177690:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830040.199917:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787830040.199926:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787830040.199945:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:0.0:1787830040.199965:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787830041.095935:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830041.095944:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787830041.095948:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830041.095957:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3476 00000100:00080000:0.0:1787830041.095965:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830041.095970:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787830041.096020:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7192c0 x1874672085965568/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830041.096048:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830041.096052:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787830041.096053:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830041.096057:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3476 00000100:00080000:0.0:1787830041.096060:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830041.096068:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830041.096078:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830041.096092:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718f00 x1874672085965696/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830041.096103:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830041.096106:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787830041.096109:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787830041.096110:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085965568:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1787830041.096115:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3476 00000100:00080000:0.0:1787830041.096120:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830041.096124:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830041.096128:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830041.096139:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a940 x1874672085965824/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830041.096153:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830041.096156:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787830041.096157:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830041.096160:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3476 00000100:00080000:0.0:1787830041.096163:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830041.096165:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830041.096169:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:0.0:1787830041.096178:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718780 x1874672085965952/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787830041.096183:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787830041.096188:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830041.096191:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787830041.096192:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830041.096195:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3476 00000100:00080000:0.0:1787830041.096197:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830041.096199:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830041.096202:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:0.0:1787830041.096208:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da719a40 x1874672085966080/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830041.096213:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830041.096214:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787830041.096215:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830041.096218:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3476 00000100:00080000:0.0:1787830041.096220:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830041.096222:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830041.096225:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830041.096240:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830041/real 1787830041] req@ffff9e35da7192c0 x1874672085965568/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830066 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830041.096247:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7183c0 x1874672085966208/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830041.096261:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830041.096265:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:2.0:1787830041.096267:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7192c0 x1874672085965568/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830066 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830041.096268:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830041.096277:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3476 00000100:00080000:0.0:1787830041.096282:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830041.096283:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830041.096287:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830041.096292:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1787830041.096299:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787830041.096306:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085965568:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1787830041.096307:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718000 x1874672085966336/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830041.096319:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830041.096322:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787830041.096324:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830041.096328:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3476 00000100:00100000:2.0:1787830041.096336:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085965696:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787830041.096340:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830041.096347:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830041.096352:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:0.0:1787830041.096364:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a580 x1874672085966464/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787830041.096368:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787830041.096383:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830041.096387:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787830041.096388:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830041.096392:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3476 00000100:00080000:0.0:1787830041.096395:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787830041.096397:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830041/real 1787830041] req@ffff9e35da718f00 x1874672085965696/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830066 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830041.096401:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830041.096409:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:0.0:1787830041.096430:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718b40 x1874672085966592/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787830041.096603:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672085965696/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830066 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830041.096610:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830041.096612:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787830041.096615:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085965696:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787830041.096619:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085965824:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787830041.096636:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787830041.096644:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830041/real 1787830041] req@ffff9e35da71a940 x1874672085965824/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830066 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830041.096655:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672085965824/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830066 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830041.096662:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830041.096665:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787830041.096668:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085965824:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787830041.096674:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085965952:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787830041.096692:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787830041.096700:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830041/real 1787830041] req@ffff9e35da718780 x1874672085965952/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830066 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830041.096708:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718780 x1874672085965952/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830066 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830041.096715:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830041.096718:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787830041.096721:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085965952:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787830041.096727:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085966080:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787830041.096745:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787830041.096756:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830041/real 1787830041] req@ffff9e35da719a40 x1874672085966080/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830066 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830041.096767:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672085966080/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830066 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830041.096782:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830041.096785:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787830041.096788:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085966080:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787830041.096793:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085966208:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787830041.096837:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787830041.096846:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830041/real 1787830041] req@ffff9e35da7183c0 x1874672085966208/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830066 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830041.096857:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672085966208/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830066 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830041.096865:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830041.096868:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787830041.096871:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085966208:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787830041.096876:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085966336:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787830041.096906:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787830041.096920:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830041/real 1787830041] req@ffff9e35da718000 x1874672085966336/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830066 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830041.096928:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718000 x1874672085966336/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830066 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830041.096933:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830041.096935:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787830041.096937:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085966336:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787830041.096954:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085966464:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787830041.096971:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787830041.096977:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830041/real 1787830041] req@ffff9e35da71a580 x1874672085966464/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830066 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830041.097007:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672085966464/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830066 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830041.097015:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830041.097018:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787830041.097020:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085966464:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787830041.097023:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085966592:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787830041.097037:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787830041.097044:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830041/real 1787830041] req@ffff9e35da718b40 x1874672085966592/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830066 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830041.097056:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718b40 x1874672085966592/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830066 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830041.097063:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830041.097065:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787830041.097068:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085966592:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830041.223935:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787830041.224016:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830041.224548:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:3.0:1787830041.224909:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:3.0:1787830042.248465:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787830042.248590:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830042.249036:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830042.249279:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787830043.272055:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787830043.272897:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:0.0:1787830043.273178:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830043.273228:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830044.295928:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830044.295928:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787830044.295938:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787830044.295980:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830045.319940:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830045.320030:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787830045.320174:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787830045.320415:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00080000:1.0:1787830046.216271:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830046.216283:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787830046.216287:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830046.216298:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3481 00000100:00080000:1.0:1787830046.216309:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830046.216316:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787830046.216355:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6d00 x1874672085966720/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787830046.216427:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085966720:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787830046.216497:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830046.216501:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787830046.216503:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830046.216507:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3481 00000100:00080000:1.0:1787830046.216511:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830046.216515:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830046.216525:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830046.216540:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4780 x1874672085966848/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830046.216551:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830046.216555:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787830046.216557:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830046.216561:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3481 00000100:00080000:1.0:1787830046.216564:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830046.216568:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830046.216573:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830046.216583:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6580 x1874672085966976/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830046.216592:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830046.216595:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787830046.216597:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830046.216601:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3481 00000100:00080000:1.0:1787830046.216604:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830046.216607:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830046.216612:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:1.0:1787830046.216622:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7480 x1874672085967104/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830046.216631:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830046.216634:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787830046.216636:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830046.216639:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3481 00000100:00080000:1.0:1787830046.216642:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830046.216645:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830046.216649:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:1.0:1787830046.216658:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7840 x1874672085967232/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830046.216666:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830046.216669:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787830046.216671:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830046.216674:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3481 00000100:00080000:1.0:1787830046.216677:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830046.216680:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830046.216684:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830046.216694:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4f00 x1874672085967360/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830046.216702:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830046.216705:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787830046.216707:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830046.216710:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3481 00000100:00080000:1.0:1787830046.216713:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830046.216717:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830046.216721:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830046.216730:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f61c0 x1874672085967488/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830046.216738:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830046.216741:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787830046.216743:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830046.216747:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3481 00000100:00080000:1.0:1787830046.216749:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830046.216753:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830046.216757:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:1.0:1787830046.216767:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6940 x1874672085967616/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830046.216774:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830046.216776:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787830046.216778:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830046.216781:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3481 00000100:00080000:1.0:1787830046.216784:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830046.216787:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830046.216790:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:1.0:1787830046.216832:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f70c0 x1874672085967744/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787830046.216911:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787830046.216977:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830046/real 1787830046] req@ffff9e35da6f6d00 x1874672085966720/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830071 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830046.216995:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6d00 x1874672085966720/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830071 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830046.217005:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830046.217009:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787830046.217012:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085966720:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1787830046.217035:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085966848:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787830046.217060:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787830046.217222:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830046/real 1787830046] req@ffff9e35da6f4780 x1874672085966848/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830071 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830046.217239:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4780 x1874672085966848/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830071 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830046.217246:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830046.217248:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787830046.217251:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085966848:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787830046.217257:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085966976:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787830046.217279:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787830046.217290:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830046/real 1787830046] req@ffff9e35da6f6580 x1874672085966976/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830071 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830046.217306:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6580 x1874672085966976/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830071 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830046.217313:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830046.217316:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787830046.217318:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085966976:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787830046.217323:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085967104:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787830046.217342:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787830046.217352:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830046/real 1787830046] req@ffff9e35da6f7480 x1874672085967104/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830071 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830046.217362:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7480 x1874672085967104/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830071 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830046.217371:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830046.217374:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787830046.217377:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085967104:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787830046.217382:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085967232:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787830046.217397:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787830046.217405:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830046/real 1787830046] req@ffff9e35da6f7840 x1874672085967232/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830071 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830046.217414:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7840 x1874672085967232/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830071 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830046.217425:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830046.217427:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787830046.217429:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085967232:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787830046.217433:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085967360:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787830046.217447:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787830046.217456:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830046/real 1787830046] req@ffff9e35da6f4f00 x1874672085967360/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830071 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830046.217466:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4f00 x1874672085967360/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830071 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830046.217531:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830046.217536:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787830046.217540:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085967360:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787830046.217546:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085967488:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787830046.217567:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787830046.217577:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830046/real 1787830046] req@ffff9e35da6f61c0 x1874672085967488/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830071 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830046.217587:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f61c0 x1874672085967488/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830071 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830046.217594:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830046.217597:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787830046.217599:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085967488:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787830046.217604:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085967616:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787830046.217620:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787830046.217628:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830046/real 1787830046] req@ffff9e35da6f6940 x1874672085967616/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830071 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830046.217639:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6940 x1874672085967616/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830071 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830046.217646:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830046.217648:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787830046.217651:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085967616:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787830046.217656:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085967744:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787830046.217672:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787830046.217681:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830046/real 1787830046] req@ffff9e35da6f70c0 x1874672085967744/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830071 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830046.217690:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f70c0 x1874672085967744/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830071 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830046.217696:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830046.217698:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787830046.217701:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085967744:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830046.343939:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787830046.344015:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787830046.344106:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830046.344176:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830047.367927:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787830047.367927:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787830047.367959:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787830047.368000:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830048.391921:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787830048.391925:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:1.0:1787830048.392105:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830048.392657:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787830049.415940:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830049.415941:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787830049.415993:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787830049.416067:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830050.439945:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787830050.441961:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787830050.441987:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787830050.442017:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00080000:3.0:1787830051.335973:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830051.335985:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787830051.335990:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830051.336002:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3486 00000100:00080000:3.0:1787830051.336013:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830051.336020:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787830051.336055:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b480 x1874672085967872/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830051.336078:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830051.336081:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787830051.336083:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830051.336087:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3486 00000100:00080000:3.0:1787830051.336091:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830051.336095:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830051.336105:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830051.336118:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498780 x1874672085968000/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830051.336126:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830051.336129:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:2.0:1787830051.336130:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085967872:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1787830051.336131:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830051.336138:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3486 00000100:00080000:3.0:1787830051.336142:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830051.336144:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830051.336148:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830051.336155:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498f00 x1874672085968128/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830051.336161:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830051.336163:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787830051.336164:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830051.336167:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3486 00000100:00080000:3.0:1787830051.336169:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830051.336171:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830051.336173:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:3.0:1787830051.336180:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498000 x1874672085968256/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830051.336186:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830051.336188:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787830051.336189:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830051.336192:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3486 00000100:00080000:3.0:1787830051.336194:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830051.336196:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830051.336198:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:3.0:1787830051.336204:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499a40 x1874672085968384/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830051.336210:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830051.336213:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787830051.336215:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830051.336219:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3486 00000100:00080000:3.0:1787830051.336224:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830051.336227:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830051.336231:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830051.336242:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a1c0 x1874672085968512/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787830051.336250:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085968000:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1787830051.336254:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830051.336259:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787830051.336262:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830051.336268:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3486 00000100:00080000:3.0:1787830051.336275:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830051.336279:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830051.336283:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830051.336284:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085968128:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787830051.336297:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b840 x1874672085968640/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830051.336307:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830051.336311:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787830051.336314:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830051.336320:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3486 00000100:00080000:3.0:1787830051.336322:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787830051.336327:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085968256:192.168.203.141@tcp:8:kworker.0 00000100:00080000:3.0:1787830051.336327:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830051.336334:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:3.0:1787830051.336346:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649bc00 x1874672085968768/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830051.336354:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830051.336356:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787830051.336359:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830051.336362:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3486 00000100:00080000:3.0:1787830051.336364:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787830051.336367:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085968384:192.168.203.141@tcp:8:kworker.0 00000100:00080000:3.0:1787830051.336369:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830051.336375:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:3.0:1787830051.336389:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498b40 x1874672085968896/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787830051.336403:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085968512:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787830051.336419:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085968640:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787830051.336430:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085968768:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787830051.336440:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085968896:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787830051.463962:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787830051.463963:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787830051.464010:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:0.0:1787830051.464036:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830052.487940:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830052.487941:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787830052.488031:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830052.488457:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787830053.511951:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787830053.511951:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830053.511994:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787830053.512021:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1787830054.408213:0:2396:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.141:988 00000400:00000100:1.0:1787830054.408263:0:2396:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.141@tcp at host 192.168.203.141 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1787830054.408279:0:2396:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:1.0:1787830054.408319:0:2396:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787830054.408353:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830051/real 1787830054] req@ffff9e35c6498b40 x1874672085968896/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830076 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830054.408390:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498b40 x1874672085968896/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830076 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830054.408402:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830054.408407:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787830054.408411:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085968896:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787830054.408420:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830051/real 1787830054] req@ffff9e35c649b480 x1874672085967872/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830076 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830054.408433:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b480 x1874672085967872/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830076 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830054.408441:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830054.408444:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787830054.408448:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085967872:192.168.203.141@tcp:250:kworker.0 00000800:00000100:1.0:1787830054.409017:0:2396:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:1.0:1787830054.409044:0:2396:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:1.0:1787830054.409489:0:2396:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:1.0:1787830054.409499:0:2396:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:1.0:1787830054.409513:0:2396:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:1.0:1787830054.409522:0:2396:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:1.0:1787830054.409528:0:2396:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:2.0:1787830054.409857:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830051/real 1787830054] req@ffff9e35c649a1c0 x1874672085968512/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830076 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830054.409878:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a1c0 x1874672085968512/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830076 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830054.409899:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830054.409903:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787830054.409907:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085968512:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787830054.409914:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830051/real 1787830054] req@ffff9e35c649b840 x1874672085968640/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830076 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830054.409926:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b840 x1874672085968640/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830076 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830054.409934:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830054.409936:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787830054.409939:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085968640:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787830054.409944:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830051/real 1787830054] req@ffff9e35c649bc00 x1874672085968768/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830076 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830054.409956:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649bc00 x1874672085968768/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830076 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830054.409963:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830054.409966:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787830054.409969:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085968768:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787830054.409975:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830051/real 1787830054] req@ffff9e35c6498780 x1874672085968000/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830076 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830054.409986:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498780 x1874672085968000/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830076 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830054.409993:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830054.409996:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787830054.409999:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085968000:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787830054.410005:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830051/real 1787830054] req@ffff9e35c6498f00 x1874672085968128/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830076 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830054.410015:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498f00 x1874672085968128/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830076 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830054.410022:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830054.410025:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787830054.410028:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085968128:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787830054.410034:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830051/real 1787830054] req@ffff9e35c6498000 x1874672085968256/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830076 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830054.410047:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498000 x1874672085968256/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830076 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830054.410054:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830054.410057:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787830054.410060:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085968256:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787830054.410065:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830051/real 1787830054] req@ffff9e35c6499a40 x1874672085968384/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830076 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830054.410075:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499a40 x1874672085968384/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830076 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830054.410082:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830054.410084:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787830054.410087:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085968384:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830054.535926:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787830054.535951:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787830054.535965:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787830054.535973:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830055.560020:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:0.0:1787830055.560117:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787830055.560286:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830055.560368:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787830056.455990:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830056.456000:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787830056.456004:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830056.456014:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3491 00000100:00080000:0.0:1787830056.456023:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830056.456029:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787830056.456060:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a1c0 x1874672085969024/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830056.456090:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830056.456094:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787830056.456096:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830056.456099:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3491 00000100:00080000:0.0:1787830056.456104:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830056.456107:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830056.456117:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830056.456151:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718b40 x1874672085969152/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830056.456161:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830056.456164:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787830056.456166:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830056.456170:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3491 00000100:00080000:0.0:1787830056.456174:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830056.456177:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830056.456182:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830056.456193:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a580 x1874672085969280/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830056.456201:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830056.456204:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787830056.456206:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830056.456209:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3491 00000100:00080000:0.0:1787830056.456213:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830056.456216:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830056.456220:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:0.0:1787830056.456230:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718000 x1874672085969408/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830056.456238:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830056.456241:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787830056.456243:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830056.456259:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3491 00000100:00080000:0.0:1787830056.456266:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830056.456270:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830056.456274:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:0.0:1787830056.456286:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7183c0 x1874672085969536/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830056.456301:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830056.456304:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787830056.456306:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830056.456312:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3491 00000100:00100000:2.0:1787830056.456314:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085969024:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1787830056.456315:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830056.456332:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830056.456337:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1787830056.456403:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830056.456418:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da719a40 x1874672085969664/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830056.456431:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830056.456434:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787830056.456436:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830056.456440:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3491 00000100:00080000:0.0:1787830056.456444:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830056.456448:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830056.456453:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830056.456460:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830056/real 1787830056] req@ffff9e35da71a1c0 x1874672085969024/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830081 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830056.456467:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718780 x1874672085969792/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830056.456484:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830056.456487:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787830056.456489:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830056.456495:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3491 00000100:00100000:2.0:1787830056.456496:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a1c0 x1874672085969024/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830081 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830056.456500:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830056.456507:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830056.456514:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00080000:2.0:1787830056.456519:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830056.456527:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830056.456528:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a940 x1874672085969920/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787830056.456537:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085969024:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1787830056.456542:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830056.456546:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787830056.456549:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787830056.456552:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085969152:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787830056.456556:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3491 00000100:00080000:0.0:1787830056.456563:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830056.456567:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830056.456572:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:0.0:1787830056.456585:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718f00 x1874672085970048/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787830056.456593:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787830056.456616:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830056/real 1787830056] req@ffff9e35da718b40 x1874672085969152/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830081 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830056.456632:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718b40 x1874672085969152/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830081 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830056.456641:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830056.456645:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787830056.456648:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085969152:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787830056.456655:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085969280:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787830056.456674:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787830056.456685:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830056/real 1787830056] req@ffff9e35da71a580 x1874672085969280/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830081 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830056.456697:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672085969280/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830081 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830056.456705:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830056.456709:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787830056.456712:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085969280:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787830056.456718:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085969408:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787830056.456736:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787830056.456746:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830056/real 1787830056] req@ffff9e35da718000 x1874672085969408/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830081 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830056.456758:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718000 x1874672085969408/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830081 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830056.456765:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830056.456769:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787830056.456772:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085969408:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787830056.456779:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085969536:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787830056.456826:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787830056.456840:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830056/real 1787830056] req@ffff9e35da7183c0 x1874672085969536/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830081 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830056.456852:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672085969536/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830081 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830056.456862:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830056.456865:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787830056.456869:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085969536:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787830056.456920:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085969664:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787830056.456943:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787830056.456955:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830056/real 1787830056] req@ffff9e35da719a40 x1874672085969664/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830081 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830056.456972:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672085969664/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830081 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830056.456980:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830056.456983:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787830056.456986:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085969664:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787830056.456991:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085969792:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787830056.457006:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787830056.457015:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830056/real 1787830056] req@ffff9e35da718780 x1874672085969792/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830081 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830056.457025:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718780 x1874672085969792/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830081 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830056.457032:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830056.457034:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787830056.457037:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085969792:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787830056.457042:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085969920:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787830056.457058:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787830056.457070:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830056/real 1787830056] req@ffff9e35da71a940 x1874672085969920/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830081 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830056.457081:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672085969920/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830081 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830056.457096:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830056.457100:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787830056.457103:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085969920:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787830056.457108:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085970048:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787830056.457127:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787830056.457137:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830056/real 1787830056] req@ffff9e35da718f00 x1874672085970048/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830081 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830056.457148:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672085970048/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830081 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830056.457157:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830056.457161:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787830056.457164:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085970048:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787830056.583929:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830056.584180:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787830056.584461:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787830056.584915:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787830057.607932:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787830057.607946:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830057.608506:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830057.608847:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787830058.631963:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787830058.631963:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830058.631995:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:0.0:1787830058.632407:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830059.655958:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:3.0:1787830059.656941:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:1.0:1787830059.657324:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787830059.657331:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787830060.680191:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787830060.680328:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830060.680434:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830060.680785:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00080000:0.0:1787830061.575942:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830061.575952:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787830061.575955:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830061.575963:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3496 00000100:00080000:0.0:1787830061.575973:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830061.575980:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787830061.576026:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7192c0 x1874672085970176/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787830061.576099:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085970176:192.168.203.141@tcp:250:kworker.0 00000800:00000100:2.0:1787830061.576159:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787830061.576196:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830061.576201:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787830061.576203:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830061.576207:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3496 00000100:00080000:0.0:1787830061.576211:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830061.576215:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830061.576224:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830061.576237:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718f00 x1874672085970304/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787830061.576334:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830061/real 1787830061] req@ffff9e35da7192c0 x1874672085970176/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830086 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830061.576351:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7192c0 x1874672085970176/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830086 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830061.576366:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830061.576370:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787830061.576373:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085970176:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1787830061.576397:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085970304:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787830061.576421:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787830061.576433:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830061/real 1787830061] req@ffff9e35da718f00 x1874672085970304/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830086 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830061.576443:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672085970304/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830086 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830061.576451:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830061.576453:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787830061.576456:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085970304:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787830061.576557:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830061.576561:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787830061.576563:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830061.576568:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3496 00000100:00080000:0.0:1787830061.576572:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830061.576576:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830061.576582:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830061.576594:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a940 x1874672085970432/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787830061.576639:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085970432:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787830061.576662:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787830061.576682:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830061.576686:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787830061.576688:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830061.576692:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3496 00000100:00080000:0.0:1787830061.576696:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830061.576701:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830061.576705:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:0.0:1787830061.576718:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718780 x1874672085970560/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787830061.576751:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830061/real 1787830061] req@ffff9e35da71a940 x1874672085970432/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830086 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830061.576759:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672085970432/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830086 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830061.576765:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830061.576767:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787830061.576769:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085970432:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787830061.576778:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085970560:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787830061.576792:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787830061.576818:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830061/real 1787830061] req@ffff9e35da718780 x1874672085970560/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830086 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830061.576826:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718780 x1874672085970560/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830086 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830061.576831:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830061.576832:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787830061.576834:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085970560:192.168.203.141@tcp:8:kworker.0 00000100:00080000:0.0:1787830061.576931:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830061.576936:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787830061.576938:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830061.576943:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3496 00000100:00080000:0.0:1787830061.576947:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830061.576951:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830061.576957:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:0.0:1787830061.576968:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da719a40 x1874672085970688/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830061.576989:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830061.576993:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787830061.576995:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830061.576998:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3496 00000100:00080000:0.0:1787830061.577002:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830061.577006:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830061.577011:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830061.577022:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7183c0 x1874672085970816/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830061.577030:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830061.577033:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787830061.577034:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830061.577038:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3496 00000100:00080000:0.0:1787830061.577041:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787830061.577042:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085970688:192.168.203.141@tcp:8:kworker.0 00000100:00080000:0.0:1787830061.577044:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830061.577048:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830061.577057:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718000 x1874672085970944/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787830061.577063:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787830061.577066:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830061.577069:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787830061.577071:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787830061.577073:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830061/real 1787830061] req@ffff9e35da719a40 x1874672085970688/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830086 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830061.577074:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3496 00000100:00080000:0.0:1787830061.577077:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830061.577080:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787830061.577084:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672085970688/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830086 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1787830061.577085:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:0.0:1787830061.577093:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a580 x1874672085971072/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830061.577093:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830061.577096:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:0.0:1787830061.577101:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830061.577104:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:2.0:1787830061.577104:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085970688:192.168.203.141@tcp:8:kworker.0 00000100:00080000:0.0:1787830061.577105:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830061.577109:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3496 00000100:00100000:2.0:1787830061.577110:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085970816:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787830061.577112:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830061.577115:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830061.577119:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000800:00000100:2.0:1787830061.577126:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830061.577127:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718b40 x1874672085971200/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787830061.577222:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830061/real 1787830061] req@ffff9e35da7183c0 x1874672085970816/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830086 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830061.577235:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672085970816/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830086 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830061.577240:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830061.577242:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787830061.577244:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085970816:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787830061.577255:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085970944:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787830061.577270:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787830061.577277:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830061/real 1787830061] req@ffff9e35da718000 x1874672085970944/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830086 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830061.577283:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718000 x1874672085970944/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830086 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830061.577288:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830061.577290:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787830061.577292:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085970944:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787830061.577295:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085971072:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787830061.577303:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787830061.577308:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830061/real 1787830061] req@ffff9e35da71a580 x1874672085971072/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830086 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830061.577314:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672085971072/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830086 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830061.577322:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830061.577324:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787830061.577326:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085971072:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787830061.577328:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085971200:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787830061.577337:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787830061.577341:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830061/real 1787830061] req@ffff9e35da718b40 x1874672085971200/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830086 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830061.577358:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718b40 x1874672085971200/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830086 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830061.577366:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830061.577369:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787830061.577372:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085971200:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787830061.703932:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787830061.704042:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:3.0:1787830061.704057:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:0.0:1787830061.704317:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830062.728010:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787830062.729956:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787830062.730236:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830062.730262:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787830063.751936:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830063.751937:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787830063.751990:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830063.752195:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787830064.775930:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:0.0:1787830064.776561:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787830064.776588:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787830064.776898:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787830065.799972:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787830065.800252:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787830065.801092:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830065.802458:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00080000:0.0:1787830066.696227:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830066.696237:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787830066.696242:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830066.696251:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3502 00000100:00080000:0.0:1787830066.696312:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830066.696319:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787830066.696355:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a1c0 x1874672085971328/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830066.696382:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830066.696385:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787830066.696386:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830066.696390:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3502 00000100:00080000:0.0:1787830066.696394:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830066.696397:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830066.696406:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830066.696418:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718b40 x1874672085971456/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830066.696426:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830066.696437:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787830066.696439:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830066.696443:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3502 00000100:00080000:0.0:1787830066.696446:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830066.696449:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830066.696453:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830066.696461:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a580 x1874672085971584/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830066.696468:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830066.696470:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787830066.696472:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830066.696475:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3502 00000100:00080000:0.0:1787830066.696477:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830066.696479:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830066.696483:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:0.0:1787830066.696489:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718000 x1874672085971712/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787830066.696524:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085971328:192.168.203.141@tcp:250:kworker.0 00000800:00000100:2.0:1787830066.696577:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787830066.696613:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830066/real 1787830066] req@ffff9e35da71a1c0 x1874672085971328/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830091 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830066.696630:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a1c0 x1874672085971328/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830091 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830066.696643:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830066.696647:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787830066.696650:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085971328:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1787830066.696657:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085971456:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787830066.696673:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787830066.696682:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830066/real 1787830066] req@ffff9e35da718b40 x1874672085971456/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830091 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830066.696692:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718b40 x1874672085971456/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830091 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830066.696699:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830066.696701:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787830066.696704:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085971456:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787830066.696709:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085971584:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787830066.696722:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787830066.696729:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830066/real 1787830066] req@ffff9e35da71a580 x1874672085971584/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830091 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830066.696738:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672085971584/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830091 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830066.696745:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830066.696747:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787830066.696750:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085971584:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787830066.697157:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830066.697161:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787830066.697163:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830066.697167:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3502 00000100:00080000:0.0:1787830066.697171:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830066.697175:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830066.697180:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:0.0:1787830066.697193:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7183c0 x1874672085971840/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830066.697202:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830066.697205:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787830066.697207:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830066.697210:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3502 00000100:00080000:0.0:1787830066.697213:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830066.697217:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830066.697221:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830066.697228:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da719a40 x1874672085971968/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830066.697237:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830066.697240:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787830066.697241:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830066.697244:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3502 00000100:00080000:0.0:1787830066.697247:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830066.697250:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830066.697254:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830066.697262:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718780 x1874672085972096/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830066.697318:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830066.697322:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787830066.697324:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830066.697328:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3502 00000100:00080000:0.0:1787830066.697331:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830066.697334:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830066.697338:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:2.0:1787830066.697372:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085971712:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787830066.697395:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787830066.697405:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830066/real 1787830066] req@ffff9e35da718000 x1874672085971712/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830091 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830066.697415:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718000 x1874672085971712/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830091 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830066.697421:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830066.697424:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787830066.697427:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085971712:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787830066.697431:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085971840:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787830066.697446:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830066.697450:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a940 x1874672085972224/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787830066.697458:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830066/real 1787830066] req@ffff9e35da7183c0 x1874672085971840/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830091 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830066.697467:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830066.697472:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:2.0:1787830066.697473:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672085971840/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830091 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830066.697474:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830066.697483:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3502 00000100:00080000:2.0:1787830066.697488:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830066.697490:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830066.697494:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:0.0:1787830066.697499:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787830066.697500:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085971840:192.168.203.141@tcp:8:kworker.0 00000008:00100000:0.0:1787830066.697509:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:2.0:1787830066.697510:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085971968:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787830066.697526:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787830066.697534:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830066/real 1787830066] req@ffff9e35da719a40 x1874672085971968/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830091 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830066.697543:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672085971968/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830091 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830066.697549:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830066.697552:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787830066.697554:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085971968:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787830066.697559:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085972096:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787830066.697573:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830066.697583:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718f00 x1874672085972352/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787830066.697585:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830066/real 1787830066] req@ffff9e35da718780 x1874672085972096/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830091 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830066.697613:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718780 x1874672085972096/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830091 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830066.697623:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830066.697626:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787830066.697628:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085972096:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787830066.697645:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085972224:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787830066.697658:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787830066.697666:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830066/real 1787830066] req@ffff9e35da71a940 x1874672085972224/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830091 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830066.697674:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672085972224/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830091 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830066.697686:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830066.697689:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787830066.697692:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085972224:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787830066.697696:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085972352:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787830066.697707:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787830066.697714:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830066/real 1787830066] req@ffff9e35da718f00 x1874672085972352/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830091 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830066.697723:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672085972352/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830091 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830066.697729:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830066.697732:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787830066.697734:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085972352:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787830066.823902:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787830066.823984:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787830066.824019:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830066.824055:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787830067.847927:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830067.847927:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787830067.847928:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787830067.849249:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830068.872045:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787830068.872080:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787830068.872837:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787830068.872838:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787830069.895955:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:3.0:1787830069.895955:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787830069.896036:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787830069.897691:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787830070.919921:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787830070.920058:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787830070.922358:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830070.925111:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00080000:1.0:1787830071.816607:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830071.816615:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787830071.816619:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830071.816629:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3507 00000100:00080000:1.0:1787830071.816639:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830071.816647:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787830071.816689:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4b40 x1874672085972480/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830071.816720:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830071.816723:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787830071.816725:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830071.816729:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3507 00000100:00080000:1.0:1787830071.816733:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830071.816737:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830071.816746:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830071.816758:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5e00 x1874672085972608/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830071.816778:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830071.816782:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787830071.816784:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830071.816788:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3507 00000100:00080000:1.0:1787830071.816791:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830071.816795:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830071.816823:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830071.816835:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5a40 x1874672085972736/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830071.816845:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830071.816848:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787830071.816850:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830071.816853:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3507 00000100:00080000:1.0:1787830071.816856:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830071.816860:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830071.816864:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:1.0:1787830071.816875:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f43c0 x1874672085972864/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830071.816884:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830071.816887:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787830071.816889:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830071.816893:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3507 00000100:00080000:1.0:1787830071.816896:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830071.816899:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830071.816903:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:1.0:1787830071.816911:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4000 x1874672085972992/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830071.816918:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830071.816921:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787830071.816923:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830071.816926:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3507 00000100:00080000:1.0:1787830071.816929:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830071.816931:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830071.816935:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830071.816945:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f70c0 x1874672085973120/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830071.816953:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830071.816956:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787830071.816957:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830071.816960:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3507 00000100:00080000:1.0:1787830071.816964:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830071.816967:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830071.816971:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830071.816980:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6940 x1874672085973248/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830071.816988:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830071.816991:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787830071.816992:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830071.816996:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3507 00000100:00080000:1.0:1787830071.816998:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830071.817002:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830071.817006:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:1.0:1787830071.817015:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f61c0 x1874672085973376/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830071.817022:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830071.817024:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787830071.817026:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830071.817029:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3507 00000100:00080000:1.0:1787830071.817032:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830071.817035:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830071.817039:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:1.0:1787830071.817048:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4f00 x1874672085973504/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830071.817169:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085972480:192.168.203.141@tcp:250:kworker.0 00000800:00000100:0.0:1787830071.817240:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830071.817279:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830071/real 1787830071] req@ffff9e35da6f4b40 x1874672085972480/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830096 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830071.817295:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4b40 x1874672085972480/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830096 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830071.817304:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830071.817308:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830071.817311:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085972480:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1787830071.817318:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085972608:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830071.817338:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830071.817350:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830071/real 1787830071] req@ffff9e35da6f5e00 x1874672085972608/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830096 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830071.817361:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5e00 x1874672085972608/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830096 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830071.817367:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830071.817370:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830071.817374:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085972608:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830071.817380:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085972736:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830071.817397:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830071.817407:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830071/real 1787830071] req@ffff9e35da6f5a40 x1874672085972736/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830096 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830071.817416:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5a40 x1874672085972736/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830096 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830071.817422:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830071.817425:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830071.817427:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085972736:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830071.817432:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085972864:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830071.817451:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830071.817461:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830071/real 1787830071] req@ffff9e35da6f43c0 x1874672085972864/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830096 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830071.817471:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f43c0 x1874672085972864/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830096 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830071.817492:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830071.817495:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830071.817499:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085972864:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830071.817504:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085972992:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830071.817524:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830071.817535:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830071/real 1787830071] req@ffff9e35da6f4000 x1874672085972992/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830096 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830071.817546:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4000 x1874672085972992/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830096 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830071.817554:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830071.817557:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830071.817560:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085972992:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830071.817565:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085973120:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830071.817583:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830071.817594:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830071/real 1787830071] req@ffff9e35da6f70c0 x1874672085973120/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830096 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830071.817605:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f70c0 x1874672085973120/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830096 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830071.817612:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830071.817615:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830071.817618:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085973120:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830071.817623:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085973248:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830071.817641:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830071.817652:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830071/real 1787830071] req@ffff9e35da6f6940 x1874672085973248/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830096 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830071.817663:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6940 x1874672085973248/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830096 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830071.817672:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830071.817675:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830071.817678:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085973248:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830071.817697:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085973376:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830071.817716:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830071.817727:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830071/real 1787830071] req@ffff9e35da6f61c0 x1874672085973376/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830096 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830071.817738:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f61c0 x1874672085973376/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830096 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830071.817745:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830071.817749:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830071.817752:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085973376:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830071.817757:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085973504:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830071.817788:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830071.817820:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830071/real 1787830071] req@ffff9e35da6f4f00 x1874672085973504/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830096 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830071.817833:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4f00 x1874672085973504/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830096 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830071.817840:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830071.817842:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830071.817845:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085973504:192.168.203.141@tcp:8:kworker.0 00010000:00010000:1.0:1787830071.944124:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787830071.944501:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787830071.944691:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787830071.944745:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:3.0:1787830072.967959:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787830072.968037:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830072.968637:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787830072.969317:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787830073.991938:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787830073.992542:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:1.0:1787830073.994066:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830073.994121:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787830075.015941:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787830075.016037:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787830075.016312:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:1.0:1787830075.017378:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787830076.040028:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787830076.040421:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830076.040980:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830076.041906:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00080000:3.0:1787830076.935940:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830076.935953:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787830076.935957:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830076.935967:0:27606:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3512 00000100:00080000:3.0:1787830076.936032:0:27606:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830076.936040:0:27606:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787830076.936074:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b0c0 x1874672085973632/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830076.936104:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830076.936108:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787830076.936110:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830076.936114:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3512 00000100:00080000:3.0:1787830076.936118:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830076.936122:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830076.936132:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830076.936143:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a940 x1874672085973760/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830076.936152:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830076.936155:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787830076.936157:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830076.936160:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3512 00000100:00080000:3.0:1787830076.936164:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830076.936167:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830076.936171:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830076.936180:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499680 x1874672085973888/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830076.936189:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830076.936192:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787830076.936194:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830076.936197:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3512 00000100:00080000:3.0:1787830076.936200:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830076.936302:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830076.936308:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:3.0:1787830076.936321:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a580 x1874672085974016/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830076.936332:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830076.936335:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787830076.936337:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830076.936341:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3512 00000100:00080000:3.0:1787830076.936345:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787830076.936348:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085973632:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1787830076.936351:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830076.936359:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:3.0:1787830076.936373:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499e00 x1874672085974144/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830076.936384:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830076.936388:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787830076.936390:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830076.936394:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3512 00000100:00080000:3.0:1787830076.936397:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830076.936401:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830076.936406:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1787830076.936411:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830076.936448:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830076/real 1787830076] req@ffff9e35c649b0c0 x1874672085973632/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830101 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830076.936466:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b0c0 x1874672085973632/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830101 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830076.936476:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830076.936480:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830076.936483:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085973632:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1787830076.936489:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085973760:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830076.936510:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830076.936521:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830076/real 1787830076] req@ffff9e35c649a940 x1874672085973760/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830101 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830076.936542:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a940 x1874672085973760/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830101 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830076.936551:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830076.936555:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830076.936558:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085973760:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830076.936564:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085973888:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830076.936580:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830076.936589:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830076/real 1787830076] req@ffff9e35c6499680 x1874672085973888/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830101 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830076.936600:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499680 x1874672085973888/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830101 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830076.936607:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830076.936610:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830076.936613:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085973888:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830076.936617:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085974016:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830076.936630:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830076.936638:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830076/real 1787830076] req@ffff9e35c649a580 x1874672085974016/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830101 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830076.936648:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a580 x1874672085974016/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830101 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830076.936656:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830076.936658:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830076.936661:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085974016:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830076.936687:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085974144:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787830076.937707:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499a40 x1874672085974272/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830076.937722:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830076.937725:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787830076.937726:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830076.937744:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3512 00000100:00080000:3.0:1787830076.937749:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830076.937754:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830076.937760:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830076.937770:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498000 x1874672085974400/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830076.937917:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830076.937920:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787830076.937922:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830076.937926:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3512 00000100:00080000:3.0:1787830076.937929:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830076.937933:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830076.937937:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:3.0:1787830076.937947:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498f00 x1874672085974528/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830076.937955:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830076.937958:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787830076.937959:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830076.937963:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3512 00000100:00080000:3.0:1787830076.937965:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830076.938014:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830076.938020:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:3.0:1787830076.938031:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498780 x1874672085974656/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787830076.941111:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830076.941141:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830076/real 1787830076] req@ffff9e35c6499e00 x1874672085974144/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830101 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830076.944205:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499e00 x1874672085974144/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830101 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830076.944232:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830076.944240:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830076.944247:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085974144:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830076.944286:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085974272:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830076.945913:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830076.945944:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830076/real 1787830076] req@ffff9e35c6499a40 x1874672085974272/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830101 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830076.945961:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499a40 x1874672085974272/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830101 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830076.945971:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830076.945975:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830076.945979:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085974272:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830076.945987:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085974400:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830076.946004:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830076.946012:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830076/real 1787830076] req@ffff9e35c6498000 x1874672085974400/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830101 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830076.946030:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498000 x1874672085974400/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830101 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830076.946037:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830076.946040:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830076.946043:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085974400:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830076.946049:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085974528:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830076.946063:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830076.946071:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830076/real 1787830076] req@ffff9e35c6498f00 x1874672085974528/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830101 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830076.946082:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498f00 x1874672085974528/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830101 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830076.946090:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830076.946092:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830076.946095:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085974528:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830076.946101:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085974656:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830076.946113:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830076.946121:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830076/real 1787830076] req@ffff9e35c6498780 x1874672085974656/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830101 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830076.946131:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498780 x1874672085974656/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830101 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830076.946144:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830076.946147:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830076.946149:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085974656:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830077.064039:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830077.064039:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787830077.064911:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830077.064967:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787830078.087917:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787830078.087959:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:1.0:1787830078.087978:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830078.088197:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787830079.111945:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787830079.111971:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830079.113827:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830079.113883:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787830080.139384:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830080.139477:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830080.139530:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787830080.139623:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787830081.159882:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787830081.161250:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787830081.161576:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:1.0:1787830081.161628:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787830082.056628:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830082.056639:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787830082.056642:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830082.056653:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3517 00000100:00080000:0.0:1787830082.056663:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830082.056671:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787830082.056713:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7192c0 x1874672085974784/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830082.056745:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830082.056749:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787830082.056751:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830082.056755:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3517 00000100:00080000:0.0:1787830082.056758:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830082.056761:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830082.056772:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830082.056784:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718f00 x1874672085974912/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830082.056794:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830082.056816:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787830082.056818:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830082.056821:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3517 00000100:00080000:0.0:1787830082.056824:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830082.056828:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830082.056832:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830082.056842:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a940 x1874672085975040/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830082.056851:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830082.056854:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787830082.056855:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830082.056859:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3517 00000100:00080000:0.0:1787830082.056861:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830082.056864:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830082.056869:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:0.0:1787830082.056876:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718780 x1874672085975168/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830082.056884:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830082.056887:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787830082.056889:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830082.056892:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3517 00000100:00080000:0.0:1787830082.056895:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830082.056897:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830082.056902:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:0.0:1787830082.056909:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da719a40 x1874672085975296/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830082.056930:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830082.056934:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787830082.056935:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830082.056939:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3517 00000100:00080000:0.0:1787830082.056942:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830082.056945:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830082.056949:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830082.056957:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7183c0 x1874672085975424/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830082.056966:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830082.056968:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787830082.056970:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830082.056973:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3517 00000100:00080000:0.0:1787830082.056976:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830082.056979:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830082.056983:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830082.056990:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718000 x1874672085975552/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830082.056998:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830082.057001:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787830082.057002:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830082.057006:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3517 00000100:00080000:0.0:1787830082.057008:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830082.057011:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830082.057015:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:0.0:1787830082.057023:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a580 x1874672085975680/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830082.057030:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830082.057034:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787830082.057035:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830082.057039:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3517 00000100:00080000:0.0:1787830082.057041:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830082.057044:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830082.057047:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:0.0:1787830082.057054:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718b40 x1874672085975808/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787830082.057967:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085974784:192.168.203.141@tcp:250:kworker.0 00000800:00000100:2.0:1787830082.058026:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787830082.058065:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830082/real 1787830082] req@ffff9e35da7192c0 x1874672085974784/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830107 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830082.058081:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7192c0 x1874672085974784/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830107 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830082.058092:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830082.058096:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787830082.058100:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085974784:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1787830082.058107:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085974912:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787830082.058124:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787830082.058135:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830082/real 1787830082] req@ffff9e35da718f00 x1874672085974912/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830107 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830082.059706:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672085974912/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830107 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830082.059719:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830082.059723:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787830082.059727:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085974912:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787830082.059736:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085975040:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787830082.059780:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787830082.060660:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830082/real 1787830082] req@ffff9e35da71a940 x1874672085975040/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830107 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830082.060683:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672085975040/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830107 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830082.060691:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830082.060704:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787830082.060707:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085975040:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787830082.060714:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085975168:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787830082.060741:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787830082.060762:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830082/real 1787830082] req@ffff9e35da718780 x1874672085975168/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830107 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830082.060779:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718780 x1874672085975168/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830107 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830082.060787:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830082.060790:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787830082.060812:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085975168:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787830082.060819:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085975296:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787830082.061775:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787830082.061821:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830082/real 1787830082] req@ffff9e35da719a40 x1874672085975296/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830107 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830082.062268:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672085975296/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830107 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830082.062292:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830082.062296:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787830082.062299:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085975296:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787830082.062307:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085975424:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787830082.062656:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787830082.062669:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830082/real 1787830082] req@ffff9e35da7183c0 x1874672085975424/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830107 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830082.062680:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672085975424/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830107 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830082.062688:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830082.062691:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787830082.062707:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085975424:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787830082.062713:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085975552:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1787830082.062731:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787830082.062741:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830082/real 1787830082] req@ffff9e35da718000 x1874672085975552/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830107 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830082.062752:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718000 x1874672085975552/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830107 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830082.062770:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830082.062773:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787830082.062776:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085975552:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787830082.062782:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085975680:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787830082.064428:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787830082.064558:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830082/real 1787830082] req@ffff9e35da71a580 x1874672085975680/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830107 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830082.064574:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672085975680/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830107 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830082.064584:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830082.064587:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787830082.064591:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085975680:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787830082.064598:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085975808:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1787830082.064623:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787830082.064633:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830082/real 1787830082] req@ffff9e35da718b40 x1874672085975808/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830107 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830082.064649:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718b40 x1874672085975808/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830107 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830082.064657:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830082.064660:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787830082.064663:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085975808:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787830082.183927:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787830082.183974:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830082.184024:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830082.184067:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787830083.207987:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787830083.208181:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830083.208590:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787830083.208600:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787830084.231933:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787830084.231933:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787830084.232721:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:1.0:1787830084.234766:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830085.255974:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787830085.256115:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787830085.259857:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:0.0:1787830085.260200:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830086.281073:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830086.281291:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787830086.281431:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787830086.281493:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00080000:2.0:1787830087.176426:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830087.176437:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787830087.176441:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830087.176451:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3522 00000100:00080000:2.0:1787830087.176461:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830087.176468:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787830087.176500:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c000 x1874672085975936/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830087.176827:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085975936:192.168.203.141@tcp:250:kworker.0 00000800:00000100:0.0:1787830087.176887:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1787830087.177326:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830087.177331:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787830087.177333:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830087.177338:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3522 00000100:00080000:2.0:1787830087.177343:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830087.177347:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830087.177357:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830087.177373:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498ed00 x1874672085976064/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830087.177613:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830087/real 1787830087] req@ffff9e35c498c000 x1874672085975936/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830112 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830087.177630:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c000 x1874672085975936/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830112 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830087.177640:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830087.177644:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830087.177648:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085975936:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1787830087.177680:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085976064:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830087.177707:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830087.177719:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830087/real 1787830087] req@ffff9e35c498ed00 x1874672085976064/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830112 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830087.177729:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498ed00 x1874672085976064/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830112 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830087.177738:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830087.177741:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830087.177744:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085976064:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1787830087.178426:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830087.178431:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787830087.178433:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830087.178439:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3522 00000100:00080000:2.0:1787830087.178444:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830087.178448:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830087.178456:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830087.178474:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498fc00 x1874672085976192/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830087.178484:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830087.178487:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787830087.178489:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830087.178492:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3522 00000100:00080000:2.0:1787830087.178495:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830087.178498:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830087.178502:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:2.0:1787830087.178511:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498ed00 x1874672085976320/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830087.178519:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830087.178522:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787830087.178524:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830087.178527:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3522 00000100:00080000:2.0:1787830087.178530:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830087.178533:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830087.178537:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:2.0:1787830087.178544:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c000 x1874672085976448/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830087.178558:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830087.178561:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787830087.178563:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787830087.179279:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085976192:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830087.179322:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830087.179340:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830087/real 1787830087] req@ffff9e35c498fc00 x1874672085976192/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830112 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830087.179354:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498fc00 x1874672085976192/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830112 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830087.179364:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830087.179367:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830087.179371:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085976192:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830087.179377:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085976320:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830087.179393:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830087.179401:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830087/real 1787830087] req@ffff9e35c498ed00 x1874672085976320/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830112 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830087.179412:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498ed00 x1874672085976320/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830112 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830087.179420:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830087.179423:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830087.179425:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085976320:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830087.179430:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085976448:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830087.179443:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830087.179452:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830087/real 1787830087] req@ffff9e35c498c000 x1874672085976448/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830112 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830087.179462:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c000 x1874672085976448/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830112 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830087.179470:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830087.179473:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830087.179475:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085976448:192.168.203.141@tcp:8:kworker.0 00000100:00080000:2.0:1787830087.180369:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3522 00000100:00080000:2.0:1787830087.180376:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830087.180382:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830087.180392:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830087.180415:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498de00 x1874672085976576/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830087.180440:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830087.180444:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787830087.180446:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830087.180449:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3522 00000100:00080000:2.0:1787830087.180452:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830087.180455:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830087.180459:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830087.180467:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e940 x1874672085976704/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830087.180476:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830087.180479:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787830087.180480:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830087.180484:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3522 00000100:00080000:2.0:1787830087.180487:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830087.180490:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830087.180494:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:2.0:1787830087.180501:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498d680 x1874672085976832/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830087.180509:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830087.180512:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787830087.180514:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830087.180517:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3522 00000100:00080000:2.0:1787830087.180520:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830087.180523:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830087.180526:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:2.0:1787830087.180533:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cb40 x1874672085976960/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830087.181454:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085976576:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830087.181503:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830087.181523:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830087/real 1787830087] req@ffff9e35c498de00 x1874672085976576/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830112 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830087.181537:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498de00 x1874672085976576/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830112 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830087.181546:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830087.181549:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830087.181553:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085976576:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830087.181559:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085976704:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830087.181796:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830087.181827:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830087/real 1787830087] req@ffff9e35c498e940 x1874672085976704/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830112 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830087.181839:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e940 x1874672085976704/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830112 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830087.181853:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830087.181856:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830087.181859:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085976704:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830087.181865:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085976832:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830087.181883:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830087.181892:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830087/real 1787830087] req@ffff9e35c498d680 x1874672085976832/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830112 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830087.181902:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498d680 x1874672085976832/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830112 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830087.181911:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830087.181914:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830087.181916:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085976832:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830087.181922:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085976960:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830087.181936:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830087.181944:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830087/real 1787830087] req@ffff9e35c498cb40 x1874672085976960/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830112 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830087.181954:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498cb40 x1874672085976960/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830112 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830087.181961:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830087.181964:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830087.181967:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085976960:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830087.303933:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830087.304401:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787830087.304727:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787830087.305719:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830088.327923:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:3.0:1787830088.328142:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787830088.328773:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787830088.331781:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787830089.351929:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787830089.352759:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787830089.354816:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830089.354874:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830090.375982:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830090.405352:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787830090.405672:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787830090.405794:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830091.399934:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:1.0:1787830091.466997:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830091.467017:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787830091.471097:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787830092.296179:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830092.296189:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787830092.296194:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830092.296204:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3527 00000100:00080000:1.0:1787830092.296214:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830092.296220:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787830092.296256:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7840 x1874672085977088/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830092.296285:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830092.296289:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787830092.296291:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830092.296295:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3527 00000100:00080000:1.0:1787830092.296298:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830092.296301:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830092.296311:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830092.296321:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7480 x1874672085977216/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830092.296330:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830092.296333:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787830092.296335:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830092.296338:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3527 00000100:00080000:1.0:1787830092.296341:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830092.296344:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830092.296348:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830092.296356:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6580 x1874672085977344/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830092.296364:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830092.296367:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787830092.296368:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830092.296372:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3527 00000100:00080000:1.0:1787830092.296374:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830092.296377:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830092.296380:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:1.0:1787830092.296387:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4780 x1874672085977472/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830092.296395:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830092.296397:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787830092.296399:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830092.296403:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3527 00000100:00080000:1.0:1787830092.296406:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830092.296409:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830092.296412:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:1.0:1787830092.296420:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6d00 x1874672085977600/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830092.296429:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830092.296431:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787830092.296433:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830092.296436:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3527 00000100:00080000:1.0:1787830092.296438:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830092.296440:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830092.296444:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830092.296451:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4f00 x1874672085977728/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830092.296459:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830092.296462:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787830092.296464:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830092.296467:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3527 00000100:00080000:1.0:1787830092.296469:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830092.296472:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830092.296476:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830092.296484:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f61c0 x1874672085977856/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830092.296492:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830092.296495:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787830092.296496:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830092.296500:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3527 00000100:00080000:1.0:1787830092.296503:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830092.296506:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830092.296509:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:1.0:1787830092.296516:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6940 x1874672085977984/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830092.296525:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830092.296527:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787830092.296529:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830092.296532:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3527 00000100:00080000:1.0:1787830092.296535:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830092.296538:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830092.296541:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:1.0:1787830092.296549:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f70c0 x1874672085978112/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830092.296678:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085977088:192.168.203.141@tcp:250:kworker.0 00000800:00000100:0.0:1787830092.296733:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830092.296769:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830092/real 1787830092] req@ffff9e35da6f7840 x1874672085977088/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830117 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830092.296783:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7840 x1874672085977088/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830117 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830092.296793:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830092.296796:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830092.296843:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085977088:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1787830092.296850:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085977216:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830092.296871:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830092.296882:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830092/real 1787830092] req@ffff9e35da6f7480 x1874672085977216/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830117 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830092.296893:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7480 x1874672085977216/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830117 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830092.296900:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830092.296904:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830092.296907:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085977216:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830092.296912:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085977344:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830092.296927:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830092.296936:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830092/real 1787830092] req@ffff9e35da6f6580 x1874672085977344/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830117 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830092.296946:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6580 x1874672085977344/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830117 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830092.296953:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830092.296956:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830092.296959:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085977344:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830092.296964:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085977472:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830092.296982:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830092.296992:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830092/real 1787830092] req@ffff9e35da6f4780 x1874672085977472/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830117 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830092.297001:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4780 x1874672085977472/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830117 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830092.297009:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830092.297012:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830092.297015:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085977472:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830092.297020:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085977600:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830092.297034:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830092.297042:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830092/real 1787830092] req@ffff9e35da6f6d00 x1874672085977600/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830117 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830092.297052:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6d00 x1874672085977600/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830117 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830092.297060:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830092.297063:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830092.297065:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085977600:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830092.297070:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085977728:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830092.297084:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830092.297092:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830092/real 1787830092] req@ffff9e35da6f4f00 x1874672085977728/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830117 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830092.297101:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4f00 x1874672085977728/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830117 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830092.297108:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830092.297111:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830092.297115:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085977728:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830092.297120:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085977856:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830092.297134:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830092.297143:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830092/real 1787830092] req@ffff9e35da6f61c0 x1874672085977856/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830117 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830092.297159:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f61c0 x1874672085977856/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830117 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830092.297165:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830092.297168:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830092.297171:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085977856:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830092.297176:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085977984:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830092.297190:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830092.297198:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830092/real 1787830092] req@ffff9e35da6f6940 x1874672085977984/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830117 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830092.297208:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6940 x1874672085977984/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830117 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830092.297216:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830092.297218:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830092.297221:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085977984:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830092.297226:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085978112:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830092.297239:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830092.297247:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830092/real 1787830092] req@ffff9e35da6f70c0 x1874672085978112/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830117 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830092.297257:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f70c0 x1874672085978112/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830117 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830092.297269:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830092.297272:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830092.297275:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085978112:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830092.424028:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787830092.488077:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787830092.488184:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787830092.488259:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787830093.447928:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:3.0:1787830093.511929:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787830093.511993:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787830093.512125:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830094.472552:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787830094.535943:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787830094.535988:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787830094.536142:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830095.496518:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830095.560063:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787830095.560093:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830095.560325:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787830096.520014:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830096.583957:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787830096.584277:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787830096.584365:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787830097.417221:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830097.417234:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787830097.417238:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830097.417249:0:27606:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3532 00000100:00080000:2.0:1787830097.417260:0:27606:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830097.417268:0:27606:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787830097.417316:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e580 x1874672085978240/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830097.417338:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830097.417341:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787830097.417343:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830097.417347:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3532 00000100:00080000:2.0:1787830097.417350:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830097.417354:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830097.417363:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830097.417374:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cf00 x1874672085978368/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830097.417506:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085978240:192.168.203.141@tcp:250:kworker.0 00000800:00000100:0.0:1787830097.417560:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830097.417605:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830097/real 1787830097] req@ffff9e35c498e580 x1874672085978240/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830122 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830097.417621:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e580 x1874672085978240/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830122 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830097.417630:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830097.417634:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830097.417637:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085978240:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1787830097.417904:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830097.417907:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787830097.417909:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830097.417913:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3532 00000100:00080000:2.0:1787830097.417917:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830097.417921:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830097.417926:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830097.417938:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f0c0 x1874672085978496/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830097.417948:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830097.417951:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787830097.417953:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830097.417956:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3532 00000100:00080000:2.0:1787830097.417959:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830097.417962:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830097.417965:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:2.0:1787830097.417974:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f840 x1874672085978624/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830097.417982:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830097.418453:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787830097.418455:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830097.418460:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3532 00000100:00080000:2.0:1787830097.418465:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830097.418468:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830097.418474:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:2.0:1787830097.418488:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c3c0 x1874672085978752/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830097.418497:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830097.418500:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787830097.418501:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830097.418505:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3532 00000100:00080000:2.0:1787830097.418508:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830097.418511:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830097.418516:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830097.418526:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c780 x1874672085978880/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830097.418534:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830097.418537:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787830097.418538:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830097.418542:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3532 00000100:00080000:2.0:1787830097.418545:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830097.418548:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830097.418551:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830097.418559:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f480 x1874672085979008/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830097.418567:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830097.418570:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787830097.418571:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830097.418574:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3532 00000100:00080000:2.0:1787830097.418577:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830097.418580:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830097.418584:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:2.0:1787830097.418602:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e1c0 x1874672085979136/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830097.418609:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830097.418611:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787830097.418613:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830097.418616:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3532 00000100:00080000:2.0:1787830097.418619:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830097.418621:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830097.418625:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:2.0:1787830097.418633:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e580 x1874672085979264/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830097.418743:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085978368:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830097.418781:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830097.418833:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830097/real 1787830097] req@ffff9e35c498cf00 x1874672085978368/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830122 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830097.418853:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498cf00 x1874672085978368/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830122 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830097.418862:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830097.418865:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830097.418868:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085978368:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830097.418874:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085978496:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830097.419528:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830097.419546:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830097/real 1787830097] req@ffff9e35c498f0c0 x1874672085978496/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830122 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830097.419560:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f0c0 x1874672085978496/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830122 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830097.419569:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830097.419572:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830097.419575:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085978496:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830097.419582:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085978624:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830097.420852:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830097.420874:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830097/real 1787830097] req@ffff9e35c498f840 x1874672085978624/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830122 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830097.420968:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f840 x1874672085978624/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830122 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830097.420983:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830097.421129:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830097.421134:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085978624:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830097.421143:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085978752:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830097.421176:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830097.421188:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830097/real 1787830097] req@ffff9e35c498c3c0 x1874672085978752/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830122 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830097.421210:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c3c0 x1874672085978752/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830122 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830097.421218:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830097.421221:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830097.421224:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085978752:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830097.421231:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085978880:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830097.421250:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830097.421261:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830097/real 1787830097] req@ffff9e35c498c780 x1874672085978880/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830122 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830097.421271:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c780 x1874672085978880/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830122 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830097.421278:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830097.421281:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830097.421284:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085978880:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830097.421289:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085979008:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830097.421309:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830097.421320:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830097/real 1787830097] req@ffff9e35c498f480 x1874672085979008/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830122 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830097.421329:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f480 x1874672085979008/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830122 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830097.421337:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830097.421340:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830097.421343:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085979008:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830097.421349:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085979136:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830097.421367:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830097.421378:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830097/real 1787830097] req@ffff9e35c498e1c0 x1874672085979136/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830122 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830097.421402:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e1c0 x1874672085979136/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830122 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830097.421410:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830097.421413:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830097.421416:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085979136:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830097.421422:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085979264:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830097.421441:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830097.421450:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830097/real 1787830097] req@ffff9e35c498e580 x1874672085979264/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830122 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830097.421462:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e580 x1874672085979264/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830122 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830097.421470:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830097.421473:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830097.421475:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085979264:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830097.543894:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787830097.609007:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787830097.609023:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830097.609194:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787830098.568014:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787830098.631911:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787830098.632076:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:0.0:1787830098.632741:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830099.591999:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787830099.655926:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787830099.656435:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787830099.656497:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787830100.615946:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787830100.679929:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787830100.679929:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787830100.680025:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830101.639938:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787830101.703860:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787830101.704183:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830101.704216:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00080000:3.0:1787830102.536984:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830102.536997:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787830102.537001:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830102.537012:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3537 00000100:00080000:3.0:1787830102.537135:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830102.537144:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787830102.537182:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649bc00 x1874672085979392/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830102.537214:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830102.537218:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787830102.537220:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830102.537528:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3537 00000100:00100000:0.0:1787830102.537530:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085979392:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1787830102.537535:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830102.537542:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:0.0:1787830102.537599:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:3.0:1787830102.537625:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830102.537635:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830102/real 1787830102] req@ffff9e35c649bc00 x1874672085979392/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830127 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787830102.537648:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b840 x1874672085979520/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830102.537654:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649bc00 x1874672085979392/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830127 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830102.537667:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830102.537668:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830102.537674:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:3.0:1787830102.537675:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:0.0:1787830102.537678:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085979392:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1787830102.537679:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830102.537688:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3537 00000100:00080000:3.0:1787830102.537692:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830102.537696:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830102.537701:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830102.537702:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085979520:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787830102.537714:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a1c0 x1874672085979648/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830102.537726:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830102.537729:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787830102.537731:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830102.537735:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3537 00000800:00000100:0.0:1787830102.537739:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1787830102.537740:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830102.537747:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830102.537752:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:0.0:1787830102.537756:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830102/real 1787830102] req@ffff9e35c649b840 x1874672085979520/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830127 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787830102.537765:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b480 x1874672085979776/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830102.537774:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b840 x1874672085979520/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830127 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830102.537778:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830102.537787:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787830102.537787:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787830102.537792:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830102.537793:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830102.537825:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085979520:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830102.537839:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085979648:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830102.537888:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830102.537900:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830102/real 1787830102] req@ffff9e35c649a1c0 x1874672085979648/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830127 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830102.537912:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a1c0 x1874672085979648/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830127 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830102.537919:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830102.537922:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830102.537925:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085979648:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830102.537930:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085979776:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830102.537948:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830102.537959:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830102/real 1787830102] req@ffff9e35c649b480 x1874672085979776/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830127 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830102.537975:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b480 x1874672085979776/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830127 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830102.537983:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830102.537986:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830102.537989:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085979776:192.168.203.141@tcp:8:kworker.0 00000100:00080000:3.0:1787830102.538696:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3537 00000100:00080000:3.0:1787830102.538703:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830102.538708:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830102.538716:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:3.0:1787830102.538735:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498b40 x1874672085979904/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830102.538764:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830102.538768:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787830102.538769:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830102.538773:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3537 00000100:00080000:3.0:1787830102.538777:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830102.538780:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830102.538784:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830102.538795:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c64992c0 x1874672085980032/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830102.538826:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830102.538829:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787830102.538831:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830102.538834:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3537 00000100:00080000:3.0:1787830102.538838:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830102.538841:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830102.538846:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830102.538856:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649ad00 x1874672085980160/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830102.538865:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830102.538868:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787830102.538870:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830102.538874:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3537 00000100:00080000:3.0:1787830102.538877:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830102.538880:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830102.538884:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:3.0:1787830102.539035:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c64983c0 x1874672085980288/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830102.539047:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830102.539050:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787830102.539052:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830102.539056:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3537 00000100:00080000:3.0:1787830102.539061:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830102.539064:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830102.539070:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:3.0:1787830102.539082:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b480 x1874672085980416/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830102.539727:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085979904:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830102.539773:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830102.539794:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830102/real 1787830102] req@ffff9e35c6498b40 x1874672085979904/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830127 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830102.539833:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498b40 x1874672085979904/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830127 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830102.539855:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830102.539865:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830102.539869:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085979904:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830102.539876:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c64992c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085980032:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830102.539896:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830102.539908:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830102/real 1787830102] req@ffff9e35c64992c0 x1874672085980032/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830127 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830102.539920:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c64992c0 x1874672085980032/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830127 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830102.539933:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830102.539936:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830102.539939:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c64992c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085980032:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830102.539945:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085980160:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830102.539974:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830102.539984:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830102/real 1787830102] req@ffff9e35c649ad00 x1874672085980160/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830127 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830102.539995:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649ad00 x1874672085980160/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830127 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830102.540003:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830102.540006:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830102.540009:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085980160:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830102.540015:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c64983c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085980288:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830102.540032:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830102.540042:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830102/real 1787830102] req@ffff9e35c64983c0 x1874672085980288/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830127 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830102.540052:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c64983c0 x1874672085980288/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830127 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830102.540060:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830102.540062:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830102.540066:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c64983c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085980288:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830102.540072:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085980416:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830102.540090:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830102.540113:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830102/real 1787830102] req@ffff9e35c649b480 x1874672085980416/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830127 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830102.540124:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b480 x1874672085980416/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830127 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830102.540131:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830102.540139:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830102.540142:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085980416:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830102.663921:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830102.727926:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787830102.727976:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787830102.728213:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830103.688006:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830103.751939:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787830103.752012:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787830103.753177:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830104.712633:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:0.0:1787830104.776112:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787830104.776300:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787830104.776611:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787830105.735922:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830105.799914:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:0.0:1787830105.800020:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787830105.800985:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787830106.759940:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787830106.823921:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787830106.823949:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830106.824601:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00080000:3.0:1787830107.656585:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830107.656598:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787830107.656602:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830107.656613:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3543 00000100:00080000:3.0:1787830107.656622:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830107.656629:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787830107.656665:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a1c0 x1874672085980544/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830107.656695:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830107.656699:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787830107.656700:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830107.656704:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3543 00000100:00080000:3.0:1787830107.656708:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830107.656711:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830107.656721:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830107.656733:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b840 x1874672085980672/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830107.656744:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830107.656749:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:0.0:1787830107.656750:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085980544:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1787830107.656751:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830107.656760:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3543 00000100:00080000:3.0:1787830107.656764:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830107.656767:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830107.656771:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830107.656783:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649bc00 x1874672085980800/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787830107.656851:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1787830107.656862:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830107.656866:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787830107.656869:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830107.656876:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3543 00000100:00080000:3.0:1787830107.656879:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830107.656883:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830107.656889:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:0.0:1787830107.656896:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830107/real 1787830107] req@ffff9e35c649a1c0 x1874672085980544/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830132 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787830107.656903:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498780 x1874672085980928/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830107.656920:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787830107.656924:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a1c0 x1874672085980544/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830132 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830107.656925:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787830107.656929:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830107.656938:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3543 00000100:00080000:0.0:1787830107.656943:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787830107.656944:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830107.656951:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:3.0:1787830107.656953:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787830107.656959:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085980544:192.168.203.141@tcp:250:kworker.0 00000008:00100000:3.0:1787830107.656960:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:3.0:1787830107.656977:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498f00 x1874672085981056/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830107.656989:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787830107.656992:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085980672:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1787830107.656994:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787830107.656998:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830107.657003:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3543 00000100:00080000:3.0:1787830107.657007:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830107.657012:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830107.657017:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1787830107.657023:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787830107.657031:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498000 x1874672085981184/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830107.657041:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830107/real 1787830107] req@ffff9e35c649b840 x1874672085980672/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830132 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830107.657043:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830107.657050:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787830107.657053:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830107.657062:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3543 00000100:00080000:3.0:1787830107.657067:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787830107.657068:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b840 x1874672085980672/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830132 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830107.657075:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830107.657084:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1787830107.657087:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830107.657093:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830107.657100:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085980672:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787830107.657100:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499a40 x1874672085981312/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830107.657113:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085980800:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1787830107.657116:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830107.657121:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787830107.657123:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830107.657128:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3543 00000800:00000100:0.0:1787830107.657137:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830107.657145:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830107/real 1787830107] req@ffff9e35c649bc00 x1874672085980800/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830132 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830107.657153:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649bc00 x1874672085980800/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830132 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830107.657158:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830107.657160:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830107.657162:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085980800:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830107.657167:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085980928:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830107.657180:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830107.657186:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830107/real 1787830107] req@ffff9e35c6498780 x1874672085980928/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830132 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830107.657193:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498780 x1874672085980928/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830132 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830107.657198:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830107.657200:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830107.657202:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085980928:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830107.657216:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085981056:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830107.657232:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830107.657553:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830107/real 1787830107] req@ffff9e35c6498f00 x1874672085981056/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830132 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830107.657568:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498f00 x1874672085981056/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830132 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830107.657576:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830107.657580:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830107.657583:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085981056:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830107.657589:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085981184:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830107.657611:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830107.657623:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830107/real 1787830107] req@ffff9e35c6498000 x1874672085981184/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830132 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830107.657634:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498000 x1874672085981184/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830132 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830107.657642:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830107.657645:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830107.657648:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085981184:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830107.657654:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085981312:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830107.657674:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830107.657685:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830107/real 1787830107] req@ffff9e35c6499a40 x1874672085981312/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830132 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830107.657696:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499a40 x1874672085981312/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830132 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830107.657703:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830107.657706:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830107.657708:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085981312:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1787830107.657829:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830107.657838:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830107.657844:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:3.0:1787830107.657858:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499e00 x1874672085981440/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830107.657889:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830107.657893:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787830107.657894:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830107.657898:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3543 00000100:00080000:3.0:1787830107.657902:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830107.657906:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830107.657910:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:3.0:1787830107.657922:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a580 x1874672085981568/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830107.657995:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085981440:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830107.658024:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830107.658038:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830107/real 1787830107] req@ffff9e35c6499e00 x1874672085981440/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830132 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830107.658050:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499e00 x1874672085981440/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830132 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830107.658065:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830107.658068:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830107.658071:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085981440:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830107.658077:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085981568:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830107.658097:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830107.658108:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830107/real 1787830107] req@ffff9e35c649a580 x1874672085981568/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830132 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830107.658119:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a580 x1874672085981568/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830132 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830107.658127:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830107.658130:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830107.658188:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085981568:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830107.784069:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787830107.848076:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787830107.848341:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830107.848485:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787830108.808613:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830108.871932:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787830108.872228:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787830108.873847:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830109.831928:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830109.895928:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787830109.895971:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787830109.896430:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830110.857020:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830110.919933:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787830110.919952:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:0.0:1787830110.919993:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830111.879930:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830111.943930:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787830111.943930:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:0.0:1787830111.943972:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787830112.776077:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830112.776090:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787830112.776096:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830112.776220:0:27606:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3548 00000100:00080000:2.0:1787830112.776233:0:27606:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830112.776242:0:27606:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787830112.776687:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cb40 x1874672085981696/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830112.776722:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830112.776727:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787830112.776730:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830112.776735:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3548 00000100:00080000:2.0:1787830112.776740:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830112.776745:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830112.776758:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830112.776781:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498d680 x1874672085981824/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830112.776791:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830112.776795:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787830112.776827:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830112.776832:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3548 00000100:00080000:2.0:1787830112.776836:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830112.776839:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830112.776844:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830112.776856:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e940 x1874672085981952/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830112.776865:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830112.776868:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787830112.776870:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830112.776874:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3548 00000100:00080000:2.0:1787830112.776877:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830112.776881:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830112.776887:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:2.0:1787830112.776909:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498de00 x1874672085982080/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830112.776918:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830112.776921:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787830112.776923:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830112.776926:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3548 00000100:00080000:2.0:1787830112.776929:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830112.776933:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830112.776937:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:2.0:1787830112.776947:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c000 x1874672085982208/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830112.776955:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830112.776958:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787830112.776960:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830112.776964:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3548 00000100:00080000:2.0:1787830112.776967:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830112.776971:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830112.776975:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830112.776987:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498ed00 x1874672085982336/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830112.776996:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830112.777006:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787830112.777008:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830112.777013:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3548 00000100:00080000:2.0:1787830112.777016:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830112.777020:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830112.777025:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830112.777036:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498fc00 x1874672085982464/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830112.777045:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830112.777049:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787830112.777050:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830112.777054:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3548 00000100:00080000:2.0:1787830112.777058:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830112.777062:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830112.777067:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:2.0:1787830112.777078:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e580 x1874672085982592/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830112.777086:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830112.777089:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787830112.777091:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830112.777094:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3548 00000100:00080000:2.0:1787830112.777097:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830112.777101:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830112.777105:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:2.0:1787830112.777126:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e1c0 x1874672085982720/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830112.777305:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085981696:192.168.203.141@tcp:250:kworker.0 00000800:00000100:0.0:1787830112.777389:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830112.777435:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830112/real 1787830112] req@ffff9e35c498cb40 x1874672085981696/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830137 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830112.777453:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498cb40 x1874672085981696/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830137 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830112.777464:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830112.777468:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830112.777472:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085981696:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1787830112.777488:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085981824:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830112.777514:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830112.777526:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830112/real 1787830112] req@ffff9e35c498d680 x1874672085981824/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830137 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830112.777538:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498d680 x1874672085981824/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830137 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830112.777547:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830112.777550:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830112.777553:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085981824:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830112.777568:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085981952:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830112.777584:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830112.777591:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830112/real 1787830112] req@ffff9e35c498e940 x1874672085981952/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830137 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830112.777599:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e940 x1874672085981952/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830137 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830112.777604:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830112.777606:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830112.777608:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085981952:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830112.777611:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085982080:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830112.777621:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830112.777626:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830112/real 1787830112] req@ffff9e35c498de00 x1874672085982080/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830137 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830112.777632:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498de00 x1874672085982080/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830137 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830112.777637:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830112.777638:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830112.777640:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085982080:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830112.777643:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085982208:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830112.777652:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830112.777657:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830112/real 1787830112] req@ffff9e35c498c000 x1874672085982208/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830137 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830112.777663:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c000 x1874672085982208/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830137 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830112.777672:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830112.777674:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830112.777675:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085982208:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830112.777678:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085982336:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830112.777686:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830112.777691:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830112/real 1787830112] req@ffff9e35c498ed00 x1874672085982336/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830137 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830112.777697:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498ed00 x1874672085982336/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830137 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830112.777702:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830112.777704:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830112.777706:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085982336:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830112.777709:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085982464:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830112.777717:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830112.777722:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830112/real 1787830112] req@ffff9e35c498fc00 x1874672085982464/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830137 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830112.777729:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498fc00 x1874672085982464/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830137 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830112.777733:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830112.777735:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830112.777737:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085982464:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830112.777740:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085982592:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830112.777748:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830112.777752:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830112/real 1787830112] req@ffff9e35c498e580 x1874672085982592/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830137 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830112.777758:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e580 x1874672085982592/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830137 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830112.777762:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830112.777764:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830112.777766:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085982592:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830112.777768:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085982720:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830112.777776:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830112.777780:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830112/real 1787830112] req@ffff9e35c498e1c0 x1874672085982720/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830137 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830112.777787:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e1c0 x1874672085982720/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830137 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830112.777792:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830112.777793:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830112.777794:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085982720:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830112.903940:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787830112.968024:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830112.968121:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787830112.968391:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787830113.928899:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830113.991978:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787830113.992194:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787830113.992429:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830114.951949:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830115.015937:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787830115.016010:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830115.016059:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787830115.975984:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830116.039944:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787830116.039982:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787830116.040140:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787830117.000997:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787830117.064011:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787830117.064098:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830117.064115:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787830117.896007:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830117.896016:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787830117.896019:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830117.896027:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3553 00000100:00080000:1.0:1787830117.896035:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830117.896041:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787830117.896078:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4000 x1874672085982848/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830117.896109:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830117.896113:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787830117.896115:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830117.896119:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3553 00000100:00080000:1.0:1787830117.896123:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830117.896126:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830117.896143:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830117.896157:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f43c0 x1874672085982976/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830117.896165:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830117.896168:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787830117.896169:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830117.896173:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3553 00000100:00080000:1.0:1787830117.896176:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830117.896179:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830117.896183:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830117.896193:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5a40 x1874672085983104/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830117.896200:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830117.896202:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787830117.896203:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830117.896206:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3553 00000100:00080000:1.0:1787830117.896208:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830117.896210:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830117.896213:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:1.0:1787830117.896219:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5e00 x1874672085983232/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830117.896229:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830117.896231:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787830117.896232:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830117.896235:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3553 00000100:00080000:1.0:1787830117.896236:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830117.896238:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830117.896240:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:1.0:1787830117.896246:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4b40 x1874672085983360/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830117.896252:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085982848:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787830117.896254:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830117.896260:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787830117.896262:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830117.896265:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3553 00000100:00080000:1.0:1787830117.896267:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830117.896268:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830117.896271:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830117.896279:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f70c0 x1874672085983488/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830117.896285:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830117.896295:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787830117.896296:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830117.896303:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3553 00000100:00080000:1.0:1787830117.896307:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830117.896310:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830117.896314:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830117.896324:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6940 x1874672085983616/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830117.896333:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830117.896336:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787830117.896337:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830117.896341:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3553 00000100:00080000:1.0:1787830117.896345:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830117.896351:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787830117.896352:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085982976:192.168.203.141@tcp:38:kworker.0 00000008:00100000:1.0:1787830117.896357:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:1.0:1787830117.896371:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f61c0 x1874672085983744/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830117.896378:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830117.896381:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787830117.896382:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830117.896385:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3553 00000100:00100000:0.0:1787830117.896389:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085983104:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787830117.896390:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830117.896396:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830117.896400:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:1.0:1787830117.896412:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4f00 x1874672085983872/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830117.896422:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085983232:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830117.896477:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085983360:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830117.896501:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085983488:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830117.896522:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085983616:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830117.896542:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085983744:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830117.896559:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085983872:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830118.023940:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787830118.087954:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787830118.088089:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787830118.089747:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787830119.047924:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787830119.111942:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787830119.111974:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830119.112028:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787830120.071933:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830120.135934:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787830120.135934:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787830120.136329:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1787830120.968218:0:2397:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.141:988 00000400:00000100:2.0:1787830120.968265:0:2397:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.141@tcp at host 192.168.203.141 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1787830120.968280:0:2397:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:2.0:1787830120.968323:0:2397:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000800:00000100:2.0:1787830120.968335:0:2397:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:2.0:1787830120.968345:0:2397:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:2.0:1787830120.968354:0:2397:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:0.0:1787830120.968363:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830117/real 1787830120] req@ffff9e35da6f4f00 x1874672085983872/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830142 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:2.0:1787830120.968363:0:2397:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:2.0:1787830120.968374:0:2397:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:2.0:1787830120.968395:0:2397:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:0.0:1787830120.968401:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4f00 x1874672085983872/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830142 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:2.0:1787830120.968409:0:2397:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00080000:0.0:1787830120.968421:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830120.968442:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830120.968446:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085983872:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830120.968457:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830117/real 1787830120] req@ffff9e35da6f4000 x1874672085982848/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830142 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830120.968470:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4000 x1874672085982848/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830142 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830120.968477:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830120.968480:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830120.968483:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085982848:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1787830120.968488:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830117/real 1787830120] req@ffff9e35da6f43c0 x1874672085982976/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830142 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830120.968498:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f43c0 x1874672085982976/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830142 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830120.968574:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830120.968578:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830120.968582:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085982976:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830120.968588:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830117/real 1787830120] req@ffff9e35da6f5a40 x1874672085983104/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830142 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830120.968600:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5a40 x1874672085983104/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830142 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830120.968607:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830120.968610:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830120.968613:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085983104:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830120.968619:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830117/real 1787830120] req@ffff9e35da6f5e00 x1874672085983232/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830142 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830120.968629:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5e00 x1874672085983232/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830142 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830120.968655:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830120.968659:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830120.968662:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085983232:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830120.968701:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830117/real 1787830120] req@ffff9e35da6f4b40 x1874672085983360/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830142 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830120.968717:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4b40 x1874672085983360/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830142 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830120.968726:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830120.968729:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830120.968732:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085983360:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830120.968738:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830117/real 1787830120] req@ffff9e35da6f70c0 x1874672085983488/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830142 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830120.968750:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f70c0 x1874672085983488/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830142 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830120.968757:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830120.968760:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830120.968773:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085983488:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830120.968780:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830117/real 1787830120] req@ffff9e35da6f6940 x1874672085983616/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830142 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830120.968791:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6940 x1874672085983616/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830142 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830120.968836:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830120.968847:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830120.968851:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085983616:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830120.968857:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830117/real 1787830120] req@ffff9e35da6f61c0 x1874672085983744/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830142 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830120.968868:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f61c0 x1874672085983744/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830142 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830120.968876:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830120.968879:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830120.968882:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085983744:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830121.096108:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:1.0:1787830121.160045:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830121.162084:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787830121.162106:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787830122.120343:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830122.183941:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787830122.184054:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830122.184333:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00080000:3.0:1787830123.015983:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830123.015995:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787830123.015999:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830123.016010:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3558 00000100:00080000:3.0:1787830123.016020:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830123.016032:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787830123.016065:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499680 x1874672085984000/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830123.016098:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830123.016103:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787830123.016105:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830123.016109:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3558 00000100:00080000:3.0:1787830123.016112:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830123.016115:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830123.016125:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830123.016137:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a940 x1874672085984128/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830123.016145:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830123.016148:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787830123.016150:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830123.016153:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3558 00000100:00080000:3.0:1787830123.016156:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830123.016159:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830123.016163:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830123.016173:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b0c0 x1874672085984256/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830123.016181:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830123.016184:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787830123.016186:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830123.016190:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3558 00000100:00080000:3.0:1787830123.016193:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830123.016196:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830123.016201:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:3.0:1787830123.016211:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a580 x1874672085984384/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830123.016220:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830123.016222:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787830123.016224:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830123.016227:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3558 00000100:00080000:3.0:1787830123.016230:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830123.016234:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830123.016238:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:3.0:1787830123.016248:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499e00 x1874672085984512/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830123.016256:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830123.016259:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787830123.016260:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830123.016264:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3558 00000100:00080000:3.0:1787830123.016266:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830123.016269:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830123.016274:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830123.016284:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499a40 x1874672085984640/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830123.016292:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830123.016295:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787830123.016297:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830123.016300:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3558 00000100:00080000:3.0:1787830123.016304:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830123.016307:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830123.016311:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830123.016321:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498000 x1874672085984768/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830123.016329:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830123.016332:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787830123.016334:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830123.016338:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3558 00000100:00080000:3.0:1787830123.016341:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830123.016344:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830123.016348:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:3.0:1787830123.016357:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498f00 x1874672085984896/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830123.016365:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830123.016368:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787830123.016369:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830123.016373:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3558 00000100:00080000:3.0:1787830123.016375:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830123.016378:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830123.016382:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:3.0:1787830123.016391:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498780 x1874672085985024/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830123.016521:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085984000:192.168.203.141@tcp:250:kworker.0 00000800:00000100:0.0:1787830123.016580:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830123.016618:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830123/real 1787830123] req@ffff9e35c6499680 x1874672085984000/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830148 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830123.016632:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499680 x1874672085984000/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830148 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830123.016641:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830123.016645:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830123.016648:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085984000:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1787830123.016654:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085984128:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830123.016674:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830123.016685:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830123/real 1787830123] req@ffff9e35c649a940 x1874672085984128/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830148 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830123.016695:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a940 x1874672085984128/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830148 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830123.016703:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830123.016706:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830123.016708:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085984128:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830123.016714:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085984256:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830123.016732:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830123.016741:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830123/real 1787830123] req@ffff9e35c649b0c0 x1874672085984256/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830148 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830123.016752:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b0c0 x1874672085984256/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830148 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830123.016759:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830123.016762:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830123.016765:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085984256:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830123.016770:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085984384:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830123.016788:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830123.016916:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830123/real 1787830123] req@ffff9e35c649a580 x1874672085984384/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830148 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830123.016929:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a580 x1874672085984384/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830148 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830123.016937:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830123.016940:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830123.016943:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085984384:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830123.016948:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085984512:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830123.016967:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830123.016978:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830123/real 1787830123] req@ffff9e35c6499e00 x1874672085984512/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830148 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830123.016994:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499e00 x1874672085984512/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830148 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830123.017002:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830123.017005:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830123.017007:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085984512:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830123.017012:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085984640:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830123.017030:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830123.017040:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830123/real 1787830123] req@ffff9e35c6499a40 x1874672085984640/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830148 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830123.017051:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499a40 x1874672085984640/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830148 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830123.017058:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830123.017062:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830123.017065:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085984640:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830123.017070:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085984768:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830123.017089:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830123.017099:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830123/real 1787830123] req@ffff9e35c6498000 x1874672085984768/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830148 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830123.017109:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498000 x1874672085984768/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830148 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830123.017120:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830123.017123:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830123.017126:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085984768:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830123.017131:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085984896:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830123.017149:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830123.017159:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830123/real 1787830123] req@ffff9e35c6498f00 x1874672085984896/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830148 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830123.017169:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498f00 x1874672085984896/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830148 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830123.017177:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830123.017179:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830123.017182:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085984896:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830123.017188:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085985024:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830123.017204:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830123.017213:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830123/real 1787830123] req@ffff9e35c6498780 x1874672085985024/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830148 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830123.017224:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498780 x1874672085985024/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830148 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830123.017231:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830123.017234:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830123.017237:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085985024:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830123.143945:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830123.207893:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787830123.208006:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787830123.208161:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830124.169539:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830124.231922:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787830124.231935:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787830124.231953:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830125.191946:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830125.255935:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787830125.256083:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830125.256104:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787830126.215950:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:1.0:1787830126.280614:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830126.280751:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787830126.280847:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787830127.239947:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830127.304478:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787830127.304562:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787830127.304861:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00080000:1.0:1787830128.136773:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830128.136785:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787830128.136789:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830128.136827:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3563 00000100:00080000:1.0:1787830128.136850:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830128.136857:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787830128.136892:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6d00 x1874672085985152/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830128.136917:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830128.136920:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787830128.136922:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830128.136926:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3563 00000100:00080000:1.0:1787830128.136929:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830128.136933:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830128.136942:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830128.136954:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4780 x1874672085985280/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830128.136961:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085985152:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787830128.136966:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830128.136971:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787830128.136972:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830128.136976:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3563 00000100:00080000:1.0:1787830128.136980:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830128.136984:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830128.136988:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830128.136999:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6580 x1874672085985408/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830128.137008:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830128.137011:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787830128.137013:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830128.137016:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3563 00000100:00080000:1.0:1787830128.137020:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1787830128.137022:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787830128.137027:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830128.137032:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:1.0:1787830128.137044:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7480 x1874672085985536/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830128.137052:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830128.137056:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787830128.137057:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830128.137066:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3563 00000100:00080000:1.0:1787830128.137071:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830128.137077:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787830128.137078:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830128/real 1787830128] req@ffff9e35da6f6d00 x1874672085985152/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830153 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1787830128.137083:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:1.0:1787830128.137099:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7840 x1874672085985664/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830128.137103:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6d00 x1874672085985152/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830153 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830128.137117:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830128.137120:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787830128.137123:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830128.137131:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3563 00000100:00080000:0.0:1787830128.137133:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830128.137136:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830128.137140:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:1.0:1787830128.137144:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787830128.137147:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085985152:192.168.203.141@tcp:250:kworker.0 00000008:00100000:1.0:1787830128.137151:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830128.137166:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f61c0 x1874672085985792/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830128.137174:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085985280:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787830128.137176:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830128.137182:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787830128.137184:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830128.137189:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3563 00000100:00080000:1.0:1787830128.137196:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1787830128.137201:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787830128.137204:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830128.137211:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830128.137217:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830128/real 1787830128] req@ffff9e35da6f4780 x1874672085985280/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830153 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830128.137226:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6940 x1874672085985920/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830128.137233:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4780 x1874672085985280/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830153 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830128.137238:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830128.137247:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787830128.137250:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830128.137252:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830128.137257:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3563 00000100:00080000:0.0:1787830128.137258:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830128.137264:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085985280:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787830128.137264:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830128.137273:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787830128.137275:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085985408:192.168.203.141@tcp:38:kworker.0 00000008:00100000:1.0:1787830128.137283:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000800:00000100:0.0:1787830128.137297:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830128.137299:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f70c0 x1874672085986048/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830128.137309:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830128/real 1787830128] req@ffff9e35da6f6580 x1874672085985408/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830153 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830128.137310:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830128.137319:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787830128.137323:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830128.137331:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3563 00000100:00100000:0.0:1787830128.137333:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6580 x1874672085985408/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830153 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830128.137338:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830128.137347:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830128.137347:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:0.0:1787830128.137354:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000008:00100000:1.0:1787830128.137355:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:0.0:1787830128.137367:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085985408:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787830128.137369:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4b40 x1874672085986176/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830128.137386:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085985536:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830128.137412:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830128.137424:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830128/real 1787830128] req@ffff9e35da6f7480 x1874672085985536/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830153 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830128.137436:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7480 x1874672085985536/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830153 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830128.137444:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830128.137446:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830128.137450:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085985536:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830128.137456:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085985664:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830128.137471:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830128.137479:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830128/real 1787830128] req@ffff9e35da6f7840 x1874672085985664/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830153 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830128.137489:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7840 x1874672085985664/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830153 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830128.137498:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830128.137501:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830128.137504:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085985664:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830128.137519:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085985792:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830128.137533:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830128.137542:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830128/real 1787830128] req@ffff9e35da6f61c0 x1874672085985792/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830153 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830128.137553:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f61c0 x1874672085985792/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830153 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830128.137561:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830128.137564:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830128.137567:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085985792:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830128.137572:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085985920:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830128.137584:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830128.137594:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830128/real 1787830128] req@ffff9e35da6f6940 x1874672085985920/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830153 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830128.137604:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6940 x1874672085985920/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830153 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830128.137612:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830128.137614:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830128.137617:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085985920:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830128.137622:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085986048:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830128.137648:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830128.137657:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830128/real 1787830128] req@ffff9e35da6f70c0 x1874672085986048/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830153 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830128.137668:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f70c0 x1874672085986048/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830153 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830128.137675:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830128.137678:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830128.137681:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085986048:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830128.137686:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085986176:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830128.137699:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830128.137706:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830128/real 1787830128] req@ffff9e35da6f4b40 x1874672085986176/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830153 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830128.137717:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4b40 x1874672085986176/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830153 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830128.137724:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830128.137738:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830128.137742:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085986176:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830128.263929:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830128.328112:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787830128.328335:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787830128.329554:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830129.287949:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830129.351941:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787830129.352065:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:0.0:1787830129.352093:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830130.313579:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787830130.376395:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830130.376557:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787830130.376629:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787830131.336169:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:3.0:1787830131.399950:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787830131.400400:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830131.402455:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787830132.359945:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830132.423895:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787830132.425160:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:0.0:1787830132.425298:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787830133.255972:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830133.255991:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787830133.255997:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830133.256009:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3568 00000100:00080000:3.0:1787830133.256020:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830133.256027:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787830133.256063:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649bc00 x1874672085986304/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830133.256094:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830133.256096:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787830133.256097:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830133.256101:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3568 00000100:00080000:3.0:1787830133.256103:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830133.256106:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830133.256115:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830133.256124:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b840 x1874672085986432/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830133.256130:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830133.256132:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787830133.256133:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830133.256136:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3568 00000100:00080000:3.0:1787830133.256138:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830133.256141:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830133.256144:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830133.256151:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a1c0 x1874672085986560/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830133.256160:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787830133.256165:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085986304:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1787830133.256165:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787830133.256170:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830133.256176:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3568 00000100:00080000:3.0:1787830133.256180:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830133.256184:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830133.256189:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:3.0:1787830133.256202:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b480 x1874672085986688/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830133.256211:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830133.256215:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787830133.256217:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830133.256220:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3568 00000100:00080000:3.0:1787830133.256224:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1787830133.256226:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1787830133.256231:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830133.256236:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:3.0:1787830133.256248:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c64983c0 x1874672085986816/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830133.256258:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830133.256261:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787830133.256263:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830133.256267:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3568 00000100:00080000:3.0:1787830133.256271:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787830133.256274:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830133/real 1787830133] req@ffff9e35c649bc00 x1874672085986304/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830158 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830133.256278:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830133.256286:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830133.256299:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649bc00 x1874672085986304/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830158 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830133.256309:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1787830133.256314:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649ad00 x1874672085986944/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830133.256317:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830133.256325:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085986304:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1787830133.256334:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787830133.256349:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085986432:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1787830133.256352:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787830133.256356:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830133.256361:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3568 00000100:00080000:3.0:1787830133.256365:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830133.256369:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830133.256374:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1787830133.256380:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787830133.256389:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c64992c0 x1874672085987072/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830133.256394:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830133/real 1787830133] req@ffff9e35c649b840 x1874672085986432/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830158 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830133.256405:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b840 x1874672085986432/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830158 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830133.256421:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787830133.256422:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830133.256426:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787830133.256427:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830133.256433:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3568 00000100:00080000:0.0:1787830133.256437:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:3.0:1787830133.256439:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787830133.256444:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085986432:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1787830133.256447:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830133.256455:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:3.0:1787830133.256466:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498b40 x1874672085987200/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830133.256473:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830133.256474:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787830133.256476:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830133.256478:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3568 00000100:00080000:3.0:1787830133.256481:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787830133.256482:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085986560:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1787830133.256487:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830133.256493:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:3.0:1787830133.256507:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b840 x1874672085987328/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787830133.256514:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830133.256533:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830133/real 1787830133] req@ffff9e35c649a1c0 x1874672085986560/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830158 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830133.256547:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a1c0 x1874672085986560/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830158 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830133.256556:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830133.256560:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830133.256567:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085986560:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830133.256573:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085986688:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830133.256593:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830133.256604:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830133/real 1787830133] req@ffff9e35c649b480 x1874672085986688/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830158 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830133.256615:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b480 x1874672085986688/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830158 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830133.256623:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830133.256626:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830133.256629:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085986688:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830133.256634:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c64983c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085986816:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830133.256654:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830133.256664:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830133/real 1787830133] req@ffff9e35c64983c0 x1874672085986816/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830158 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830133.256675:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c64983c0 x1874672085986816/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830158 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830133.256684:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830133.256687:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830133.256690:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c64983c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085986816:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830133.256696:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085986944:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830133.256713:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830133.256723:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830133/real 1787830133] req@ffff9e35c649ad00 x1874672085986944/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830158 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830133.256735:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649ad00 x1874672085986944/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830158 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830133.256743:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830133.256745:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830133.256749:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085986944:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830133.256753:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c64992c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085987072:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830133.256767:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830133.256777:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830133/real 1787830133] req@ffff9e35c64992c0 x1874672085987072/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830158 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830133.256787:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c64992c0 x1874672085987072/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830158 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830133.256794:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830133.256834:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830133.256839:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c64992c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085987072:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830133.256845:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085987200:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830133.256865:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830133.256875:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830133/real 1787830133] req@ffff9e35c6498b40 x1874672085987200/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830158 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830133.256896:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498b40 x1874672085987200/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830158 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830133.256904:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830133.256908:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830133.256911:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085987200:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830133.256932:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085987328:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830133.256951:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830133.256963:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830133/real 1787830133] req@ffff9e35c649b840 x1874672085987328/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830158 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830133.256974:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b840 x1874672085987328/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830158 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830133.257000:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830133.257003:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830133.257006:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085987328:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830133.383945:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830133.447923:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787830133.447987:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787830133.447994:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830134.408039:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:3.0:1787830134.471875:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787830134.472009:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830134.472047:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787830135.431917:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830135.495937:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787830135.495994:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:0.0:1787830135.496064:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830136.455942:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830136.519949:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787830136.521106:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787830136.521405:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787830137.480144:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830137.544046:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787830137.544069:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787830137.544305:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787830138.375992:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830138.376005:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787830138.376010:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830138.376021:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3573 00000100:00080000:3.0:1787830138.376031:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830138.376038:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787830138.376072:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649bc00 x1874672085987456/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830138.376492:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085987456:192.168.203.141@tcp:250:kworker.0 00000800:00000100:0.0:1787830138.376563:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1787830138.376948:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830138.376953:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787830138.376956:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830138.376961:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3573 00000100:00100000:0.0:1787830138.376995:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830138/real 1787830138] req@ffff9e35c649bc00 x1874672085987456/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830163 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830138.377014:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649bc00 x1874672085987456/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830163 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830138.377023:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830138.377028:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830138.377032:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085987456:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1787830138.378971:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830138.378992:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830138.379007:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830138.379038:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498780 x1874672085987584/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830138.379069:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830138.379073:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787830138.379076:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830138.379082:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3573 00000100:00080000:3.0:1787830138.379091:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830138.379095:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830138.379102:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830138.379115:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498f00 x1874672085987712/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830138.379127:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085987584:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1787830138.379127:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830138.379136:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787830138.379139:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830138.379143:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3573 00000100:00080000:3.0:1787830138.379147:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830138.379151:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830138.379156:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:3.0:1787830138.379169:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498000 x1874672085987840/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830138.379177:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830138.379181:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787830138.379182:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830138.379186:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3573 00000100:00080000:3.0:1787830138.379191:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1787830138.379192:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1787830138.379197:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830138.379203:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:3.0:1787830138.379215:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499a40 x1874672085987968/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830138.379222:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830138/real 1787830138] req@ffff9e35c6498780 x1874672085987584/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830163 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830138.379228:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830138.379236:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787830138.379239:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830138.379246:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3573 00000100:00100000:0.0:1787830138.379250:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498780 x1874672085987584/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830163 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830138.379253:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830138.379261:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830138.379268:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1787830138.379270:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830138.379276:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787830138.379281:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499e00 x1874672085988096/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830138.379283:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085987584:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1787830138.379322:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830138.379325:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787830138.379326:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787830138.379332:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085987712:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1787830138.379332:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3573 00000100:00080000:3.0:1787830138.379339:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830138.379345:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830138.379351:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1787830138.379363:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787830138.379365:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a580 x1874672085988224/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830138.379377:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787830138.379380:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830138/real 1787830138] req@ffff9e35c6498f00 x1874672085987712/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830163 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830138.379382:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787830138.379386:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830138.379394:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3573 00000100:00080000:3.0:1787830138.379400:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787830138.379403:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498f00 x1874672085987712/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830163 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830138.379407:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830138.379415:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00080000:0.0:1787830138.379419:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830138.379424:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787830138.379428:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b0c0 x1874672085988352/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830138.379430:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085987712:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830138.379443:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085987840:192.168.203.141@tcp:8:kworker.0 00000100:00080000:3.0:1787830138.379445:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830138.379452:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787830138.379453:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830138.379457:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3573 00000100:00080000:3.0:1787830138.379461:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830138.379466:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830138.379471:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000800:00000100:0.0:1787830138.379473:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787830138.379485:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a940 x1874672085988480/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830138.379491:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830138/real 1787830138] req@ffff9e35c6498000 x1874672085987840/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830163 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830138.379509:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498000 x1874672085987840/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830163 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830138.379526:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830138.379531:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830138.379534:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085987840:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830138.379540:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085987968:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830138.379562:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830138.379573:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830138/real 1787830138] req@ffff9e35c6499a40 x1874672085987968/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830163 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830138.379584:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499a40 x1874672085987968/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830163 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830138.379593:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830138.379595:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830138.379672:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085987968:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830138.379679:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085988096:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830138.379701:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830138.379714:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830138/real 1787830138] req@ffff9e35c6499e00 x1874672085988096/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830163 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830138.379732:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499e00 x1874672085988096/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830163 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830138.379741:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830138.379743:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830138.379745:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085988096:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830138.379763:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085988224:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830138.379817:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830138.379830:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830138/real 1787830138] req@ffff9e35c649a580 x1874672085988224/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830163 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830138.379840:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a580 x1874672085988224/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830163 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830138.379848:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830138.379850:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830138.379853:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085988224:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830138.379859:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085988352:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830138.379876:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830138.379886:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830138/real 1787830138] req@ffff9e35c649b0c0 x1874672085988352/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830163 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830138.379896:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b0c0 x1874672085988352/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830163 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830138.379908:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830138.379911:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830138.379914:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085988352:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830138.379920:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085988480:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830138.379947:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830138.379957:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830138/real 1787830138] req@ffff9e35c649a940 x1874672085988480/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830163 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830138.379967:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a940 x1874672085988480/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830163 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830138.379974:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830138.379985:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830138.379989:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085988480:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830138.504221:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830138.567972:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787830138.568011:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787830138.568063:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787830139.527947:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787830139.591929:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:0.0:1787830139.592183:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830139.593268:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787830140.551953:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830140.616377:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787830140.616449:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787830140.616536:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830141.576127:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830141.639935:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787830141.639983:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830141.640184:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787830142.601493:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830142.663919:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787830142.664050:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787830142.664195:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787830143.495957:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830143.495969:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787830143.495975:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830143.495988:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3578 00000100:00080000:0.0:1787830143.495998:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830143.496007:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787830143.496043:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a1c0 x1874672085988608/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830143.496083:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830143.496087:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787830143.496089:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830143.496093:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3578 00000100:00080000:0.0:1787830143.496097:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830143.496101:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830143.496111:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830143.496126:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718b40 x1874672085988736/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830143.496136:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830143.496139:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787830143.496142:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830143.496145:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3578 00000100:00080000:0.0:1787830143.496148:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830143.496152:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830143.496158:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830143.496170:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a580 x1874672085988864/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830143.496186:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830143.496189:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787830143.496191:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830143.496196:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3578 00000100:00080000:0.0:1787830143.496200:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830143.496205:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830143.496209:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:0.0:1787830143.496220:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718000 x1874672085988992/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830143.496229:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830143.496233:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787830143.496236:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830143.496240:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3578 00000100:00080000:0.0:1787830143.496243:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830143.496248:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830143.496254:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:1.0:1787830143.496257:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085988608:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1787830143.496279:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7183c0 x1874672085989120/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830143.496290:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830143.496294:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787830143.496296:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830143.496303:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3578 00000800:00000100:1.0:1787830143.496328:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787830143.496516:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830143.496523:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830143.496529:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830143.496543:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da719a40 x1874672085989248/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830143.496577:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830143.496580:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787830143.496582:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830143.496585:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3578 00000100:00080000:0.0:1787830143.496589:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830143.496592:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830143.496596:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830143.496608:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718780 x1874672085989376/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830143.496617:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830143.496621:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787830143.496623:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830143.496630:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3578 00000100:00080000:0.0:1787830143.496636:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830143.496641:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830143.496645:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:0.0:1787830143.496656:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a940 x1874672085989504/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830143.496670:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830143.496673:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787830143.496674:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830143.496678:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3578 00000100:00080000:0.0:1787830143.496682:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830143.496685:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830143.496690:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:0.0:1787830143.496699:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718f00 x1874672085989632/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787830143.497593:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830143/real 1787830143] req@ffff9e35da71a1c0 x1874672085988608/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830168 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830143.497614:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a1c0 x1874672085988608/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830168 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830143.497625:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830143.497630:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830143.497634:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085988608:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1787830143.497643:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085988736:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787830143.497678:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830143.497692:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830143/real 1787830143] req@ffff9e35da718b40 x1874672085988736/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830168 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830143.497704:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718b40 x1874672085988736/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830168 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830143.497713:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830143.497716:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830143.497719:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085988736:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787830143.497728:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085988864:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787830143.497749:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830143.497760:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830143/real 1787830143] req@ffff9e35da71a580 x1874672085988864/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830168 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830143.497771:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672085988864/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830168 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830143.497855:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830143.497859:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830143.497862:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085988864:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787830143.497870:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085988992:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787830143.497891:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830143.497901:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830143/real 1787830143] req@ffff9e35da718000 x1874672085988992/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830168 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830143.497912:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718000 x1874672085988992/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830168 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830143.497919:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830143.497922:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830143.497925:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085988992:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830143.497951:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085989120:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787830143.497969:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830143.497979:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830143/real 1787830143] req@ffff9e35da7183c0 x1874672085989120/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830168 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830143.497990:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672085989120/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830168 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830143.497999:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830143.498002:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830143.498005:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085989120:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830143.498010:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085989248:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787830143.498063:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830143.498073:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830143/real 1787830143] req@ffff9e35da719a40 x1874672085989248/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830168 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830143.498085:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672085989248/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830168 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830143.498092:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830143.498095:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830143.498098:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085989248:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787830143.498103:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085989376:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787830143.498120:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830143.498130:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830143/real 1787830143] req@ffff9e35da718780 x1874672085989376/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830168 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830143.498498:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718780 x1874672085989376/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830168 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830143.498513:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830143.498516:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830143.498519:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085989376:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787830143.498525:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085989504:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787830143.498549:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830143.498559:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830143/real 1787830143] req@ffff9e35da71a940 x1874672085989504/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830168 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830143.498571:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672085989504/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830168 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830143.498581:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830143.498584:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830143.498587:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085989504:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830143.498594:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085989632:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787830143.498613:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830143.498625:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830143/real 1787830143] req@ffff9e35da718f00 x1874672085989632/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830168 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830143.498635:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672085989632/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830168 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830143.498643:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830143.498646:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830143.498650:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085989632:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830143.624359:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830143.687937:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787830143.688014:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830143.688160:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787830144.647934:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787830144.712146:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830144.712354:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787830144.712687:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787830145.672196:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830145.736020:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787830145.736203:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830145.738375:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787830146.696032:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:3.0:1787830146.760191:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787830146.760273:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830146.760860:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787830147.720145:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:3.0:1787830147.783952:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787830147.783967:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787830147.784035:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787830148.615931:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830148.615948:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787830148.615952:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830148.615962:0:27606:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3583 00000100:00080000:0.0:1787830148.615973:0:27606:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830148.615980:0:27606:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787830148.616020:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7192c0 x1874672085989760/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830148.616058:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830148.616062:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787830148.616064:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830148.616068:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3583 00000100:00080000:0.0:1787830148.616072:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830148.616076:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830148.616086:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830148.616110:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718f00 x1874672085989888/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830148.616119:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830148.616122:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787830148.616124:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830148.616128:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3583 00000100:00080000:0.0:1787830148.616131:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830148.616134:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830148.616138:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830148.616149:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a940 x1874672085990016/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830148.616157:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830148.616161:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787830148.616162:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830148.616166:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3583 00000100:00080000:0.0:1787830148.616170:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1787830148.616391:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085989760:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1787830148.616417:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830148.616424:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:0.0:1787830148.616439:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718780 x1874672085990144/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830148.616450:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1787830148.616453:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787830148.616455:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787830148.616458:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830148.616463:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3583 00000100:00080000:0.0:1787830148.616469:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830148.616473:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830148.616479:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:0.0:1787830148.616490:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da719a40 x1874672085990272/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830148.616500:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787830148.616501:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830148/real 1787830148] req@ffff9e35da7192c0 x1874672085989760/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830173 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830148.616507:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787830148.616511:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830148.616520:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3583 00000100:00080000:0.0:1787830148.616524:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830148.616528:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787830148.616531:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7192c0 x1874672085989760/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830173 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1787830148.616534:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00080000:1.0:1787830148.616548:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1787830148.616550:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7183c0 x1874672085990400/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830148.616555:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830148.616564:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085989760:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1787830148.616579:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787830148.616580:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085989888:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787830148.616585:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787830148.616588:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830148.616593:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3583 00000100:00080000:0.0:1787830148.616596:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830148.616600:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830148.616604:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1787830148.616612:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830148.616618:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718000 x1874672085990528/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830148.616628:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787830148.616629:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830148/real 1787830148] req@ffff9e35da718f00 x1874672085989888/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830173 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830148.616634:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787830148.616638:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830148.616647:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3583 00000100:00080000:0.0:1787830148.616651:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1787830148.616654:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672085989888/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830173 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830148.616658:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830148.616667:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00080000:1.0:1787830148.616670:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830148.616676:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830148.616681:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a580 x1874672085990656/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787830148.616684:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085989888:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787830148.616698:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085990016:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787830148.616699:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830148.616706:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787830148.616708:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830148.616713:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3583 00000100:00080000:0.0:1787830148.616718:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830148.616722:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830148.616727:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000800:00000100:1.0:1787830148.616728:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830148.616741:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718b40 x1874672085990784/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787830148.616745:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830148/real 1787830148] req@ffff9e35da71a940 x1874672085990016/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830173 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830148.616767:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672085990016/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830173 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830148.616776:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830148.616780:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830148.616785:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085990016:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787830148.616835:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085990144:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787830148.616855:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830148.616866:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830148/real 1787830148] req@ffff9e35da718780 x1874672085990144/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830173 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830148.616878:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718780 x1874672085990144/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830173 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830148.616886:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830148.616889:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830148.616892:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085990144:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830148.616898:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085990272:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787830148.616916:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830148.616926:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830148/real 1787830148] req@ffff9e35da719a40 x1874672085990272/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830173 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830148.616937:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672085990272/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830173 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830148.616944:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830148.616946:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830148.616949:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085990272:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830148.616955:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085990400:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787830148.616972:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830148.616982:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830148/real 1787830148] req@ffff9e35da7183c0 x1874672085990400/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830173 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830148.616992:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672085990400/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830173 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830148.616999:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830148.617002:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830148.617004:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085990400:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787830148.617009:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085990528:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787830148.617027:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830148.617037:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830148/real 1787830148] req@ffff9e35da718000 x1874672085990528/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830173 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830148.617048:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718000 x1874672085990528/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830173 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830148.617055:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830148.617058:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830148.617061:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085990528:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787830148.617066:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085990656:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787830148.617083:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830148.617092:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830148/real 1787830148] req@ffff9e35da71a580 x1874672085990656/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830173 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830148.617102:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672085990656/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830173 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830148.617110:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830148.617113:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830148.617116:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085990656:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830148.617121:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085990784:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787830148.617138:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830148.617148:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830148/real 1787830148] req@ffff9e35da718b40 x1874672085990784/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830173 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830148.617158:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718b40 x1874672085990784/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830173 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830148.617165:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830148.617168:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830148.617171:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085990784:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830148.744372:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830148.807961:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787830148.808282:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830148.808322:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787830149.767977:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787830149.832014:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830149.833646:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787830149.833961:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787830150.792035:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:3.0:1787830150.855925:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787830150.856008:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830150.856021:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787830151.815929:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830151.879937:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787830151.880059:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830151.880372:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787830152.839935:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830152.903906:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787830152.903906:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787830152.904018:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787830153.736060:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830153.736072:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787830153.736075:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830153.736087:0:27606:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3589 00000100:00080000:3.0:1787830153.736097:0:27606:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830153.736103:0:27606:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787830153.736142:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499680 x1874672085990912/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830153.736175:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830153.736179:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787830153.736180:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830153.736184:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3589 00000100:00080000:3.0:1787830153.736188:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830153.736192:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830153.736213:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830153.736226:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a940 x1874672085991040/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830153.736235:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830153.736238:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787830153.736240:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830153.736244:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3589 00000100:00080000:3.0:1787830153.736248:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830153.736251:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830153.736255:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830153.736264:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b0c0 x1874672085991168/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830153.736273:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830153.736275:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787830153.736277:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830153.736281:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3589 00000100:00080000:3.0:1787830153.736284:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830153.736287:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830153.736291:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:3.0:1787830153.736300:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a580 x1874672085991296/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830153.736313:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830153.736316:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787830153.736318:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830153.736321:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3589 00000100:00080000:3.0:1787830153.736324:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830153.736326:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830153.736330:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:3.0:1787830153.736341:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499e00 x1874672085991424/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830153.736349:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830153.736353:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787830153.736354:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830153.736358:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3589 00000100:00080000:3.0:1787830153.736361:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830153.736365:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830153.736369:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830153.736379:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499a40 x1874672085991552/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830153.736388:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830153.736391:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787830153.736392:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830153.736396:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3589 00000100:00080000:3.0:1787830153.736399:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830153.736402:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830153.736405:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830153.736413:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498000 x1874672085991680/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830153.736420:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830153.736422:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787830153.736424:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830153.736427:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3589 00000100:00080000:3.0:1787830153.736430:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830153.736434:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830153.736437:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:3.0:1787830153.736446:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498f00 x1874672085991808/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830153.736453:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830153.736455:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787830153.736457:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830153.736460:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3589 00000100:00080000:3.0:1787830153.736462:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830153.736465:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830153.736468:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:3.0:1787830153.736477:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498780 x1874672085991936/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787830153.736578:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085990912:192.168.203.141@tcp:250:kworker.0 00000800:00000100:1.0:1787830153.736633:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830153.736669:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830153/real 1787830153] req@ffff9e35c6499680 x1874672085990912/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830178 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830153.736683:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499680 x1874672085990912/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830178 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830153.736692:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830153.736696:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830153.736699:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085990912:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1787830153.736716:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085991040:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787830153.736738:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830153.736750:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830153/real 1787830153] req@ffff9e35c649a940 x1874672085991040/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830178 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830153.736760:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a940 x1874672085991040/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830178 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830153.736768:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830153.736771:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830153.736774:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085991040:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787830153.736779:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085991168:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787830153.736850:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830153.736861:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830153/real 1787830153] req@ffff9e35c649b0c0 x1874672085991168/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830178 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830153.736872:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b0c0 x1874672085991168/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830178 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830153.736880:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830153.736882:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830153.736885:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085991168:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787830153.736891:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085991296:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787830153.736908:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830153.736917:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830153/real 1787830153] req@ffff9e35c649a580 x1874672085991296/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830178 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830153.736927:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a580 x1874672085991296/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830178 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830153.736935:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830153.736937:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830153.736940:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085991296:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830153.736945:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085991424:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787830153.736959:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830153.736967:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830153/real 1787830153] req@ffff9e35c6499e00 x1874672085991424/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830178 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830153.736983:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499e00 x1874672085991424/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830178 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830153.736991:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830153.736994:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830153.736997:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085991424:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830153.737001:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085991552:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787830153.737014:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830153.737022:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830153/real 1787830153] req@ffff9e35c6499a40 x1874672085991552/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830178 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830153.737032:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499a40 x1874672085991552/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830178 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830153.737039:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830153.737042:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830153.737045:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085991552:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787830153.737049:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085991680:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787830153.737063:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830153.737071:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830153/real 1787830153] req@ffff9e35c6498000 x1874672085991680/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830178 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830153.737081:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498000 x1874672085991680/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830178 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830153.737093:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830153.737096:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830153.737098:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085991680:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787830153.737103:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085991808:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787830153.737116:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830153.737123:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830153/real 1787830153] req@ffff9e35c6498f00 x1874672085991808/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830178 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830153.737132:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498f00 x1874672085991808/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830178 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830153.737138:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830153.737141:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830153.737144:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085991808:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830153.737148:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085991936:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787830153.737161:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830153.737169:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830153/real 1787830153] req@ffff9e35c6498780 x1874672085991936/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830178 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830153.737178:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498780 x1874672085991936/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830178 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830153.737184:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830153.737187:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830153.737190:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085991936:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830153.864258:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830153.927932:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787830153.928308:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787830153.928672:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787830154.887988:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830154.951922:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787830154.951922:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787830154.953298:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830155.912039:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787830155.976316:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830155.976459:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787830155.976671:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787830156.936090:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787830156.999982:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830157.000137:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787830157.000958:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787830157.960052:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830158.023920:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:3.0:1787830158.024048:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787830158.026476:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787830158.856092:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830158.856104:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787830158.856109:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830158.856120:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3594 00000100:00080000:1.0:1787830158.856130:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830158.856136:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787830158.856183:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5e00 x1874672085992064/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830158.856263:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085992064:192.168.203.141@tcp:250:kworker.0 00000800:00000100:0.0:1787830158.856323:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787830158.856356:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830158.856360:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787830158.856362:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830158.856366:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3594 00000100:00080000:1.0:1787830158.856371:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830158.856375:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830158.856384:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830158.856397:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5a40 x1874672085992192/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830158.856475:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830158/real 1787830158] req@ffff9e35da6f5e00 x1874672085992064/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830183 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830158.856492:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5e00 x1874672085992064/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830183 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830158.856502:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830158.856506:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830158.856509:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085992064:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1787830158.856534:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085992192:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830158.856557:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830158.856569:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830158/real 1787830158] req@ffff9e35da6f5a40 x1874672085992192/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830183 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830158.856580:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5a40 x1874672085992192/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830183 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830158.856588:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830158.856591:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830158.856594:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085992192:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787830158.863430:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830158.863441:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787830158.863445:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830158.863457:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3594 00000100:00080000:1.0:1787830158.863468:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830158.863475:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830158.863490:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830158.863528:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f43c0 x1874672085992320/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830158.863543:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830158.863546:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787830158.863548:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830158.863557:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3594 00000100:00080000:1.0:1787830158.863563:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830158.863570:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830158.863575:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:1.0:1787830158.863590:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4000 x1874672085992448/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830158.863599:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830158.863602:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787830158.863604:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830158.863608:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3594 00000100:00080000:1.0:1787830158.863611:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830158.863615:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830158.863619:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:1.0:1787830158.863628:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4f00 x1874672085992576/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830158.863643:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830158.863660:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787830158.863662:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830158.863666:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3594 00000100:00080000:1.0:1787830158.863669:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830158.863673:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830158.863677:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830158.863689:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5a40 x1874672085992704/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830158.863697:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830158.863700:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787830158.863702:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830158.863705:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3594 00000100:00080000:1.0:1787830158.863708:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830158.863711:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830158.863714:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830158.863722:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5e00 x1874672085992832/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830158.863731:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830158.863733:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787830158.863735:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830158.863738:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3594 00000100:00080000:1.0:1787830158.863741:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830158.863744:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830158.863748:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:1.0:1787830158.863756:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4b40 x1874672085992960/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830158.863763:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830158.863766:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787830158.863768:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830158.863771:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3594 00000100:00080000:1.0:1787830158.863774:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830158.863777:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830158.863780:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:1.0:1787830158.863788:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f70c0 x1874672085993088/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830158.864218:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085992320:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830158.864277:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830158.864313:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830158/real 1787830158] req@ffff9e35da6f43c0 x1874672085992320/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830183 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830158.864333:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f43c0 x1874672085992320/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830183 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830158.864348:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830158.864355:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830158.864359:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085992320:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830158.864365:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085992448:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830158.864388:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830158.864398:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830158/real 1787830158] req@ffff9e35da6f4000 x1874672085992448/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830183 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830158.864408:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4000 x1874672085992448/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830183 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830158.864416:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830158.864419:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830158.864422:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085992448:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830158.864427:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085992576:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830158.864444:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830158.864454:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830158/real 1787830158] req@ffff9e35da6f4f00 x1874672085992576/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830183 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830158.864464:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4f00 x1874672085992576/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830183 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830158.864472:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830158.864475:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830158.864478:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085992576:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830158.864484:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085992704:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830158.864511:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830158.864521:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830158/real 1787830158] req@ffff9e35da6f5a40 x1874672085992704/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830183 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830158.864531:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5a40 x1874672085992704/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830183 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830158.864539:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830158.864542:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830158.864545:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085992704:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830158.864550:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085992832:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830158.864577:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830158.864591:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830158/real 1787830158] req@ffff9e35da6f5e00 x1874672085992832/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830183 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830158.864603:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5e00 x1874672085992832/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830183 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830158.864610:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830158.864618:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830158.864621:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085992832:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830158.864627:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085992960:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830158.864712:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830158.864725:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830158/real 1787830158] req@ffff9e35da6f4b40 x1874672085992960/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830183 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830158.864737:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4b40 x1874672085992960/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830183 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830158.864746:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830158.864748:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830158.864752:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085992960:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830158.864758:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085993088:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830158.864778:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830158.864788:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830158/real 1787830158] req@ffff9e35da6f70c0 x1874672085993088/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830183 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830158.864842:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f70c0 x1874672085993088/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830183 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830158.864851:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830158.864854:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830158.864858:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085993088:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830158.983939:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830159.048056:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:3.0:1787830159.048065:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787830159.048230:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830160.008000:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830160.071955:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787830160.072063:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830160.073664:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787830161.032153:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830161.095924:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787830161.095967:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830161.096054:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787830162.056001:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830162.119960:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787830162.120492:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787830162.122505:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830163.080202:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787830163.144125:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830163.144557:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787830163.144607:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00080000:0.0:1787830163.975976:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830163.975988:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787830163.975993:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830163.976005:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3599 00000100:00080000:0.0:1787830163.976019:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830163.976028:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787830163.976076:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a1c0 x1874672085993216/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830163.976641:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830163.976645:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787830163.976647:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830163.976653:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3599 00000100:00080000:0.0:1787830163.976658:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830163.976662:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830163.976681:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830163.976700:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718b40 x1874672085993344/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830163.976719:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830163.976722:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787830163.976724:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830163.976728:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3599 00000100:00080000:0.0:1787830163.976732:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830163.976735:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830163.976741:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830163.976752:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a580 x1874672085993472/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830163.976771:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830163.976773:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787830163.976775:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830163.976779:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3599 00000100:00080000:0.0:1787830163.976782:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830163.976785:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830163.976790:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:0.0:1787830163.976875:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718000 x1874672085993600/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830163.976884:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830163.976888:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787830163.976890:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830163.976894:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3599 00000100:00080000:0.0:1787830163.976898:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830163.976902:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830163.976907:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:0.0:1787830163.976918:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7183c0 x1874672085993728/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830163.976927:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830163.976931:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787830163.976933:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830163.976937:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3599 00000100:00080000:0.0:1787830163.976940:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830163.976943:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830163.976948:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830163.976958:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da719a40 x1874672085993856/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830163.976966:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830163.976969:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787830163.976971:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830163.976974:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3599 00000100:00080000:0.0:1787830163.976978:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830163.976981:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830163.976985:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830163.976994:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718780 x1874672085993984/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830163.977003:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830163.977006:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787830163.977008:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830163.977012:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3599 00000100:00080000:0.0:1787830163.977015:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830163.977018:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830163.977023:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:0.0:1787830163.977032:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a940 x1874672085994112/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830163.977041:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830163.977044:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787830163.977046:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830163.977050:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3599 00000100:00080000:0.0:1787830163.977054:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830163.977057:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830163.977062:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:0.0:1787830163.977072:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718f00 x1874672085994240/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787830163.977202:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085993216:192.168.203.141@tcp:250:kworker.0 00000800:00000100:1.0:1787830163.977280:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830163.977329:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830163/real 1787830163] req@ffff9e35da71a1c0 x1874672085993216/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830188 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830163.977346:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a1c0 x1874672085993216/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830188 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830163.977362:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830163.977367:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830163.977371:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085993216:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1787830163.977379:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085993344:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787830163.977403:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830163.977416:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830163/real 1787830163] req@ffff9e35da718b40 x1874672085993344/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830188 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830163.977426:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718b40 x1874672085993344/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830188 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830163.977434:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830163.977437:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830163.977439:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085993344:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787830163.977446:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085993472:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787830163.977464:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830163.977474:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830163/real 1787830163] req@ffff9e35da71a580 x1874672085993472/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830188 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830163.977485:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672085993472/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830188 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830163.977492:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830163.977496:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830163.977499:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085993472:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787830163.977520:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085993600:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787830163.977541:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830163.977551:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830163/real 1787830163] req@ffff9e35da718000 x1874672085993600/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830188 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830163.977563:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718000 x1874672085993600/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830188 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830163.977581:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830163.977584:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830163.977587:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085993600:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830163.977593:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085993728:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787830163.977612:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830163.977622:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830163/real 1787830163] req@ffff9e35da7183c0 x1874672085993728/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830188 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830163.977642:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672085993728/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830188 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830163.977651:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830163.977654:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830163.977657:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085993728:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830163.977663:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085993856:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787830163.977686:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830163.977696:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830163/real 1787830163] req@ffff9e35da719a40 x1874672085993856/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830188 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830163.977707:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672085993856/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830188 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830163.977714:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830163.977717:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830163.977720:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085993856:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787830163.977725:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085993984:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787830163.978244:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830163.978261:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830163/real 1787830163] req@ffff9e35da718780 x1874672085993984/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830188 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830163.978273:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718780 x1874672085993984/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830188 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830163.978281:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830163.978284:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830163.978287:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085993984:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787830163.978294:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085994112:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787830163.978316:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830163.978326:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830163/real 1787830163] req@ffff9e35da71a940 x1874672085994112/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830188 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830163.978336:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672085994112/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830188 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830163.978350:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830163.978354:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830163.978357:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085994112:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830163.978362:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085994240:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787830163.978379:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830163.978388:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830163/real 1787830163] req@ffff9e35da718f00 x1874672085994240/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830188 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830163.978398:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672085994240/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830188 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830163.978406:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830163.978409:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830163.978411:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085994240:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830164.104226:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830164.167950:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787830164.168021:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830164.168107:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787830165.129424:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830165.191940:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787830165.192132:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787830165.194249:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830166.153321:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830166.215919:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787830166.217436:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787830166.217660:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830167.175948:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830167.239924:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787830167.240031:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787830167.240054:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830168.199991:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830168.263927:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787830168.263927:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787830168.264246:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787830169.095969:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830169.095981:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787830169.095985:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830169.095996:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3604 00000100:00080000:1.0:1787830169.096006:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830169.096013:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787830169.096051:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6940 x1874672085994368/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830169.096083:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830169.096087:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787830169.096100:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830169.096104:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3604 00000100:00080000:1.0:1787830169.096108:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830169.096112:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830169.096121:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830169.096135:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085994368:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1787830169.096136:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f61c0 x1874672085994496/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830169.096151:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830169.096154:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787830169.096156:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830169.096159:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3604 00000100:00080000:1.0:1787830169.096163:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830169.096166:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830169.096171:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830169.096183:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7840 x1874672085994624/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830169.096192:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830169.096195:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787830169.096197:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1787830169.096202:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830169.096246:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830169/real 1787830169] req@ffff9e35da6f6940 x1874672085994368/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830194 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830169.096264:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6940 x1874672085994368/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830194 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830169.096274:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830169.096278:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830169.096282:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085994368:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1787830169.096307:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085994496:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830169.096329:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830169.096341:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830169/real 1787830169] req@ffff9e35da6f61c0 x1874672085994496/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830194 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830169.096351:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f61c0 x1874672085994496/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830194 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830169.096358:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830169.096361:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830169.096364:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085994496:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830169.096369:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085994624:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830169.096387:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830169.096397:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830169/real 1787830169] req@ffff9e35da6f7840 x1874672085994624/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830194 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830169.096407:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7840 x1874672085994624/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830194 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830169.096414:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830169.096417:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830169.096420:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085994624:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787830169.096461:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3604 00000100:00080000:1.0:1787830169.096468:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830169.096473:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830169.096479:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:1.0:1787830169.096490:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7480 x1874672085994752/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830169.096510:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830169.096513:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787830169.096515:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830169.096519:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3604 00000100:00080000:1.0:1787830169.096522:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830169.096526:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830169.096530:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:1.0:1787830169.096541:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6580 x1874672085994880/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830169.096549:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830169.096552:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787830169.096553:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830169.096557:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3604 00000100:00080000:1.0:1787830169.096560:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830169.096563:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830169.096567:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830169.096647:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4780 x1874672085995008/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830169.096656:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830169.096659:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787830169.096660:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830169.096664:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3604 00000100:00080000:1.0:1787830169.096667:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830169.096671:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830169.096675:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830169.096684:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6d00 x1874672085995136/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830169.096691:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830169.096694:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787830169.096696:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830169.096699:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3604 00000100:00080000:1.0:1787830169.096702:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830169.096705:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830169.096709:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:1.0:1787830169.096719:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7840 x1874672085995264/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830169.096726:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830169.096729:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787830169.096730:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830169.096734:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3604 00000100:00080000:1.0:1787830169.096737:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830169.096741:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830169.096745:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:1.0:1787830169.096755:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f61c0 x1874672085995392/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830169.097012:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085994752:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830169.097038:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830169.097052:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830169/real 1787830169] req@ffff9e35da6f7480 x1874672085994752/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830194 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830169.097063:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7480 x1874672085994752/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830194 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830169.097070:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830169.097074:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830169.097077:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085994752:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830169.097082:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085994880:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830169.097111:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830169.097121:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830169/real 1787830169] req@ffff9e35da6f6580 x1874672085994880/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830194 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830169.097132:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6580 x1874672085994880/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830194 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830169.097140:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830169.097142:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830169.097145:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085994880:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830169.097151:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085995008:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830169.097168:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830169.097179:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830169/real 1787830169] req@ffff9e35da6f4780 x1874672085995008/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830194 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830169.097189:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4780 x1874672085995008/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830194 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830169.097197:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830169.097200:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830169.097202:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085995008:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830169.097208:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085995136:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830169.097226:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830169.097235:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830169/real 1787830169] req@ffff9e35da6f6d00 x1874672085995136/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830194 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830169.097244:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6d00 x1874672085995136/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830194 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830169.097251:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830169.097254:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830169.097257:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085995136:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830169.097262:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085995264:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830169.097277:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830169.097285:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830169/real 1787830169] req@ffff9e35da6f7840 x1874672085995264/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830194 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830169.097294:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7840 x1874672085995264/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830194 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830169.097305:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830169.097308:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830169.097311:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085995264:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830169.097316:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085995392:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830169.097333:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830169.097342:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830169/real 1787830169] req@ffff9e35da6f61c0 x1874672085995392/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830194 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830169.097353:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f61c0 x1874672085995392/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830194 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830169.097360:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830169.097364:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830169.097367:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085995392:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830169.223931:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830169.287940:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787830169.288038:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830169.288050:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787830170.247964:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830170.311904:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787830170.311923:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787830170.311952:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830171.272052:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830171.335937:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787830171.335979:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830171.336036:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787830172.296077:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830172.359926:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787830172.360021:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787830172.360035:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830173.319993:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787830173.384106:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830173.384272:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:3.0:1787830173.385292:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787830174.217224:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830174.217235:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787830174.217239:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830174.217250:0:27606:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3609 00000100:00080000:3.0:1787830174.217259:0:27606:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830174.217267:0:27606:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787830174.217302:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649bc00 x1874672085995520/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830174.217334:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830174.217338:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787830174.217339:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830174.217344:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3609 00000100:00080000:3.0:1787830174.217347:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830174.217351:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830174.217361:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830174.217373:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b840 x1874672085995648/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830174.217382:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830174.217385:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787830174.217387:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830174.217390:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3609 00000100:00080000:3.0:1787830174.217393:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830174.217396:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830174.217401:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830174.217410:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498b40 x1874672085995776/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830174.217419:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830174.217422:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787830174.217423:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830174.217427:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3609 00000100:00080000:3.0:1787830174.217429:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830174.217433:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830174.217437:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:3.0:1787830174.217445:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c64992c0 x1874672085995904/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830174.217453:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830174.217456:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787830174.217458:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830174.217462:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3609 00000100:00080000:3.0:1787830174.217464:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830174.217467:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830174.217471:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:3.0:1787830174.217481:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649ad00 x1874672085996032/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830174.217489:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830174.217492:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787830174.217494:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830174.217497:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3609 00000100:00080000:3.0:1787830174.217500:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830174.217504:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830174.217508:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830174.217516:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c64983c0 x1874672085996160/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830174.217524:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830174.217527:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787830174.217528:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830174.217532:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3609 00000100:00080000:3.0:1787830174.217534:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830174.217537:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830174.217542:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830174.217550:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b480 x1874672085996288/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830174.217558:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830174.217560:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787830174.217562:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830174.217565:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3609 00000100:00080000:3.0:1787830174.217567:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830174.217570:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830174.217574:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:3.0:1787830174.217581:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a1c0 x1874672085996416/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830174.217588:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830174.217591:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787830174.217592:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830174.217595:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3609 00000100:00080000:3.0:1787830174.217598:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830174.217601:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830174.217604:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:3.0:1787830174.217613:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498780 x1874672085996544/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830174.217759:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085995520:192.168.203.141@tcp:250:kworker.0 00000800:00000100:0.0:1787830174.217844:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830174.217885:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830174/real 1787830174] req@ffff9e35c649bc00 x1874672085995520/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830199 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830174.217897:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649bc00 x1874672085995520/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830199 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830174.217904:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830174.217906:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830174.217909:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085995520:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1787830174.217914:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085995648:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830174.217928:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830174.217935:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830174/real 1787830174] req@ffff9e35c649b840 x1874672085995648/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830199 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830174.217942:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b840 x1874672085995648/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830199 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830174.217947:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830174.217948:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830174.217950:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085995648:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830174.217953:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085995776:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830174.217962:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830174.217967:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830174/real 1787830174] req@ffff9e35c6498b40 x1874672085995776/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830199 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830174.217977:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498b40 x1874672085995776/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830199 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830174.217981:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830174.217983:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830174.217984:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085995776:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830174.217987:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c64992c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085995904:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830174.217996:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830174.218001:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830174/real 1787830174] req@ffff9e35c64992c0 x1874672085995904/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830199 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830174.218007:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c64992c0 x1874672085995904/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830199 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830174.218012:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830174.218013:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830174.218015:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c64992c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085995904:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830174.218017:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085996032:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830174.218026:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830174.218030:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830174/real 1787830174] req@ffff9e35c649ad00 x1874672085996032/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830199 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830174.218036:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649ad00 x1874672085996032/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830199 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830174.218045:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830174.218046:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830174.218048:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085996032:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830174.218051:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c64983c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085996160:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830174.218076:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830174.218091:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830174/real 1787830174] req@ffff9e35c64983c0 x1874672085996160/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830199 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830174.218101:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c64983c0 x1874672085996160/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830199 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830174.218109:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830174.218112:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830174.218114:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c64983c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085996160:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830174.218120:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085996288:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830174.218139:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830174.218149:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830174/real 1787830174] req@ffff9e35c649b480 x1874672085996288/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830199 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830174.218159:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b480 x1874672085996288/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830199 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830174.218166:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830174.218169:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830174.218173:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085996288:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830174.218178:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085996416:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830174.218196:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830174.218205:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830174/real 1787830174] req@ffff9e35c649a1c0 x1874672085996416/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830199 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830174.218215:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a1c0 x1874672085996416/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830199 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830174.218222:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830174.218226:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830174.218228:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085996416:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830174.218234:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085996544:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830174.218251:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830174.218261:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830174/real 1787830174] req@ffff9e35c6498780 x1874672085996544/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830199 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830174.218271:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498780 x1874672085996544/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830199 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830174.218277:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830174.218280:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830174.218283:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085996544:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830174.344139:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830174.407946:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:3.0:1787830174.408075:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787830174.409250:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830175.367996:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787830175.432000:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830175.432069:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787830175.432465:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787830176.392439:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787830176.455981:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787830176.456113:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787830176.457009:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787830177.415952:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:3.0:1787830177.479944:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787830177.480101:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830177.480292:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787830178.440073:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830178.503919:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787830178.504063:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830178.504328:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787830179.336192:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830179.336200:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787830179.336204:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830179.336216:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3614 00000100:00080000:3.0:1787830179.336227:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830179.336234:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787830179.336266:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498f00 x1874672085996672/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830179.336301:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830179.336305:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787830179.336307:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830179.336311:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3614 00000100:00080000:3.0:1787830179.336315:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830179.336319:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830179.336329:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830179.336341:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085996672:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1787830179.336344:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498000 x1874672085996800/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830179.336357:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830179.336360:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787830179.336363:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830179.336367:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3614 00000100:00080000:3.0:1787830179.336382:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830179.336387:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830179.336393:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830179.336407:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499a40 x1874672085996928/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787830179.336408:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1787830179.336419:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830179.336423:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787830179.336425:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830179.336440:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3614 00000100:00080000:3.0:1787830179.336444:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830179.336448:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787830179.336451:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830179/real 1787830179] req@ffff9e35c6498f00 x1874672085996672/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830204 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830179.336469:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498f00 x1874672085996672/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830204 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830179.336479:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830179.336483:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830179.336487:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085996672:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1787830179.336511:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085996800:192.168.203.141@tcp:38:kworker.0 00000008:00100000:3.0:1787830179.336522:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:3.0:1787830179.336536:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499e00 x1874672085997056/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787830179.336538:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830179.336554:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830179/real 1787830179] req@ffff9e35c6498000 x1874672085996800/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830204 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830179.336565:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498000 x1874672085996800/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830204 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830179.336582:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830179.336586:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830179.336590:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085996800:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830179.336596:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085996928:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830179.336631:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830179.336642:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830179/real 1787830179] req@ffff9e35c6499a40 x1874672085996928/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830204 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830179.336653:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499a40 x1874672085996928/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830204 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830179.336661:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830179.336664:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830179.336666:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085996928:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1787830179.337428:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830179.337432:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787830179.337434:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830179.337439:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3614 00000100:00080000:3.0:1787830179.337444:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830179.337449:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830179.337455:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:3.0:1787830179.337471:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a580 x1874672085997184/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830179.337479:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830179.337482:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787830179.337484:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830179.337487:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3614 00000100:00080000:3.0:1787830179.337489:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830179.337492:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830179.337496:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830179.337504:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b0c0 x1874672085997312/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830179.337512:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830179.337514:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787830179.337516:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830179.337518:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3614 00000100:00080000:3.0:1787830179.337521:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830179.337523:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830179.337527:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830179.337535:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a940 x1874672085997440/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830179.337543:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830179.337545:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787830179.337546:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830179.337550:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3614 00000100:00080000:3.0:1787830179.337552:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830179.337555:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830179.337558:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:3.0:1787830179.337567:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499680 x1874672085997568/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830179.337574:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830179.337576:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787830179.337578:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830179.337581:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3614 00000100:00080000:3.0:1787830179.337583:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830179.337586:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830179.337589:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:3.0:1787830179.337598:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499a40 x1874672085997696/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830179.337672:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085997056:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830179.337706:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830179.337723:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830179/real 1787830179] req@ffff9e35c6499e00 x1874672085997056/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830204 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830179.337984:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499e00 x1874672085997056/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830204 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830179.337995:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830179.337999:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830179.338002:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085997056:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830179.338010:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085997184:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830179.338047:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830179.338064:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830179/real 1787830179] req@ffff9e35c649a580 x1874672085997184/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830204 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830179.338078:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a580 x1874672085997184/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830204 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830179.338086:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830179.338090:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830179.338093:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085997184:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830179.338099:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085997312:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830179.338180:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830179.338192:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830179/real 1787830179] req@ffff9e35c649b0c0 x1874672085997312/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830204 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830179.338204:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b0c0 x1874672085997312/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830204 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830179.338212:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830179.338215:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830179.338218:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085997312:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830179.338224:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085997440:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830179.338244:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830179.338255:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830179/real 1787830179] req@ffff9e35c649a940 x1874672085997440/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830204 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830179.338265:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a940 x1874672085997440/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830204 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830179.338273:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830179.338276:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830179.338279:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085997440:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830179.338284:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085997568:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830179.338303:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830179.338313:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830179/real 1787830179] req@ffff9e35c6499680 x1874672085997568/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830204 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830179.338324:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499680 x1874672085997568/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830204 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830179.338332:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830179.338335:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830179.338338:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085997568:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830179.338344:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085997696:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830179.338362:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830179.338373:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830179/real 1787830179] req@ffff9e35c6499a40 x1874672085997696/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830204 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830179.338383:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499a40 x1874672085997696/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830204 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830179.338391:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830179.338394:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830179.338398:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085997696:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830179.463952:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830179.527926:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787830179.528202:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787830179.529156:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787830180.487959:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830180.551925:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:3.0:1787830180.551925:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787830180.552060:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830181.514518:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787830181.575976:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830181.576085:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787830181.576142:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787830182.535914:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830182.599921:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787830182.599921:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787830182.600065:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830183.559925:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830183.623923:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787830183.627379:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787830183.628040:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787830184.455932:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830184.455942:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787830184.455946:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830184.455957:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3619 00000100:00080000:3.0:1787830184.455967:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830184.455973:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787830184.456019:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498000 x1874672085997824/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830184.456049:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830184.456053:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787830184.456055:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830184.456059:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3619 00000100:00080000:3.0:1787830184.456062:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830184.456066:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830184.456075:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830184.456086:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498f00 x1874672085997952/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830184.456096:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830184.456100:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787830184.456103:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787830184.456104:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085997824:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1787830184.456109:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3619 00000100:00080000:3.0:1787830184.456114:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830184.456116:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830184.456120:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830184.456131:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498780 x1874672085998080/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830184.456139:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830184.456142:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787830184.456143:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830184.456147:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3619 00000100:00080000:3.0:1787830184.456150:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830184.456153:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830184.456157:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:3.0:1787830184.456166:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a1c0 x1874672085998208/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830184.456175:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830184.456177:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787830184.456179:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830184.456183:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3619 00000100:00080000:3.0:1787830184.456186:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830184.456189:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830184.456194:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:3.0:1787830184.456203:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b480 x1874672085998336/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830184.456211:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830184.456215:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787830184.456216:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787830184.456221:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085997952:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1787830184.456221:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3619 00000100:00080000:3.0:1787830184.456226:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830184.456231:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830184.456235:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830184.456247:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c64983c0 x1874672085998464/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830184.456256:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830184.456259:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787830184.456262:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787830184.456265:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085998080:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1787830184.456269:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3619 00000100:00080000:3.0:1787830184.456274:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830184.456278:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830184.456283:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830184.456291:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085998208:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787830184.456296:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649ad00 x1874672085998592/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830184.456305:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830184.456308:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787830184.456310:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830184.456313:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3619 00000100:00080000:3.0:1787830184.456317:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830184.456320:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830184.456324:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:3.0:1787830184.456334:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c64992c0 x1874672085998720/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830184.456342:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830184.456345:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787830184.456346:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830184.456349:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3619 00000100:00080000:3.0:1787830184.456352:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830184.456355:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830184.456360:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:3.0:1787830184.456367:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498b40 x1874672085998848/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830184.456759:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085998336:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830184.457023:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c64983c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085998464:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830184.457046:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085998592:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830184.457062:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c64992c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085998720:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830184.457077:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085998848:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830184.584483:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830184.647909:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787830184.647939:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787830184.647949:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830185.607993:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830185.671925:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787830185.671966:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787830185.672528:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830186.631949:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830186.695941:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787830186.696341:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787830186.697433:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1787830187.528696:0:2398:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.141:988 00000400:00000100:3.0:1787830187.528745:0:2398:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.141@tcp at host 192.168.203.141 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1787830187.528761:0:2398:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:0.0:1787830187.532418:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830184/real 1787830187] req@ffff9e35c6498000 x1874672085997824/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830209 ref 2 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830187.532452:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498000 x1874672085997824/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830209 ref 2 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830187.532461:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830187.532464:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830187.532466:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085997824:192.168.203.141@tcp:250:kworker.0 00000800:00000100:3.0:1787830187.532701:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000800:00000100:3.0:1787830187.532736:0:2398:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1787830187.532746:0:2398:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:0.0:1787830187.532839:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830184/real 1787830187] req@ffff9e35c6498b40 x1874672085998848/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830209 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830187.532856:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498b40 x1874672085998848/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830209 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830187.532866:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830187.532870:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830187.532873:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085998848:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830187.532879:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830184/real 1787830187] req@ffff9e35c6498f00 x1874672085997952/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830209 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830187.532890:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498f00 x1874672085997952/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830209 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830187.532904:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830187.532907:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830187.532911:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085997952:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830187.532917:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830184/real 1787830187] req@ffff9e35c6498780 x1874672085998080/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830209 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830187.532928:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498780 x1874672085998080/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830209 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830187.532935:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830187.532938:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830187.532941:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085998080:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787830187.533306:0:2398:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:0.0:1787830187.533405:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830184/real 1787830187] req@ffff9e35c649a1c0 x1874672085998208/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830209 ref 2 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830187.533416:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a1c0 x1874672085998208/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830209 ref 2 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830187.533429:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830187.533432:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830187.533434:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085998208:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787830187.533699:0:2398:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1787830187.533713:0:2398:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1787830187.533721:0:2398:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1787830187.533727:0:2398:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:0.0:1787830187.533742:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830184/real 1787830187] req@ffff9e35c64983c0 x1874672085998464/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830209 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830187.534021:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c64983c0 x1874672085998464/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830209 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830187.534033:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830187.534038:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830187.534042:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c64983c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085998464:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830187.534048:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830184/real 1787830187] req@ffff9e35c649ad00 x1874672085998592/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830209 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830187.534061:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649ad00 x1874672085998592/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830209 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830187.534068:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830187.534072:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830187.534075:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085998592:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830187.534080:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830184/real 1787830187] req@ffff9e35c64992c0 x1874672085998720/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830209 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830187.534090:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c64992c0 x1874672085998720/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830209 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830187.534098:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830187.534101:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830187.534104:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c64992c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085998720:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830187.534110:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830184/real 1787830187] req@ffff9e35c649b480 x1874672085998336/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830209 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830187.534120:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b480 x1874672085998336/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830209 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830187.534129:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830187.534132:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830187.534135:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085998336:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830187.658112:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830187.719932:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787830187.719984:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830187.720079:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787830188.679962:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830188.743930:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787830188.744245:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830188.744525:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787830189.575935:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830189.575958:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787830189.575963:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830189.575978:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3624 00000100:00080000:3.0:1787830189.575988:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830189.575995:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787830189.576028:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a1c0 x1874672085998976/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830189.576062:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830189.576066:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787830189.576068:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830189.576073:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3624 00000100:00080000:3.0:1787830189.576076:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830189.576080:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830189.576090:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830189.576103:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498000 x1874672085999104/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830189.576112:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830189.576115:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787830189.576117:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830189.576121:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3624 00000100:00080000:3.0:1787830189.576124:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830189.576128:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830189.576133:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830189.576142:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b840 x1874672085999232/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830189.576152:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830189.576155:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787830189.576157:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830189.576172:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3624 00000100:00080000:3.0:1787830189.576176:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830189.576180:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830189.576185:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:3.0:1787830189.576196:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649bc00 x1874672085999360/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830189.576218:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830189.576223:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787830189.576224:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830189.576228:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3624 00000100:00080000:3.0:1787830189.576231:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830189.576235:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830189.576240:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:3.0:1787830189.576251:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b480 x1874672085999488/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830189.576260:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830189.576263:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787830189.576265:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830189.576268:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3624 00000100:00080000:3.0:1787830189.576272:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830189.576275:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830189.576279:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830189.576288:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c64992c0 x1874672085999616/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830189.576296:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830189.576298:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787830189.576300:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830189.576303:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3624 00000100:00080000:3.0:1787830189.576307:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830189.576310:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830189.576314:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830189.576324:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649ad00 x1874672085999744/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830189.576332:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830189.576335:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787830189.576336:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830189.576340:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3624 00000100:00080000:3.0:1787830189.576343:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830189.576347:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830189.576351:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:3.0:1787830189.576361:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c64983c0 x1874672085999872/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830189.576368:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830189.576371:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787830189.576373:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830189.576376:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3624 00000100:00080000:3.0:1787830189.576379:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830189.576382:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830189.576386:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:3.0:1787830189.576395:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498780 x1874672086000000/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830189.576490:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085998976:192.168.203.141@tcp:250:kworker.0 00000800:00000100:0.0:1787830189.576545:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830189.576581:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830189/real 1787830189] req@ffff9e35c649a1c0 x1874672085998976/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830214 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830189.576597:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a1c0 x1874672085998976/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830214 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830189.576605:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830189.576609:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830189.576613:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672085998976:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1787830189.576619:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085999104:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830189.576640:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830189.576652:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830189/real 1787830189] req@ffff9e35c6498000 x1874672085999104/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830214 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830189.576662:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498000 x1874672085999104/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830214 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830189.576674:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830189.576678:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830189.576680:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085999104:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830189.576686:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085999232:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830189.576706:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830189.576717:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830189/real 1787830189] req@ffff9e35c649b840 x1874672085999232/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830214 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830189.576727:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b840 x1874672085999232/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830214 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830189.576734:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830189.576737:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830189.576740:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085999232:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830189.576745:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085999360:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830189.576762:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830189.576773:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830189/real 1787830189] req@ffff9e35c649bc00 x1874672085999360/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830214 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830189.576784:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649bc00 x1874672085999360/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830214 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830189.576791:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830189.576793:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830189.576823:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085999360:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830189.576829:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085999488:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830189.576849:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830189.576860:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830189/real 1787830189] req@ffff9e35c649b480 x1874672085999488/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830214 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830189.576871:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b480 x1874672085999488/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830214 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830189.576879:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830189.576881:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830189.576884:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672085999488:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830189.576889:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c64992c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085999616:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830189.576906:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830189.576916:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830189/real 1787830189] req@ffff9e35c64992c0 x1874672085999616/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830214 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830189.576927:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c64992c0 x1874672085999616/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830214 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830189.576935:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830189.576938:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830189.576941:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c64992c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085999616:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830189.579365:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085999744:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830189.579448:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830189.579480:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830189/real 1787830189] req@ffff9e35c649ad00 x1874672085999744/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830214 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830189.580363:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649ad00 x1874672085999744/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830214 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830189.580378:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830189.580383:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830189.580387:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085999744:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830189.580397:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c64983c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085999872:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830189.580436:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830189.580451:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830189/real 1787830189] req@ffff9e35c64983c0 x1874672085999872/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830214 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830189.580462:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c64983c0 x1874672085999872/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830214 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830189.580471:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830189.580475:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830189.580478:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c64983c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672085999872:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830189.580484:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086000000:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830189.580579:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830189.580590:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830189/real 1787830189] req@ffff9e35c6498780 x1874672086000000/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830214 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830189.580601:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498780 x1874672086000000/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830214 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830189.580609:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830189.580613:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830189.580615:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086000000:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830189.704496:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830189.767940:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787830189.768144:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787830189.768615:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787830190.728077:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830190.791929:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787830190.791985:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787830190.791988:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787830191.754664:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830191.816168:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787830191.816646:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787830191.816683:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830192.779052:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830192.839965:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787830192.840252:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:0.0:1787830192.840394:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830193.799928:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830193.863942:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787830193.863959:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787830193.863981:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787830194.695991:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830194.696001:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787830194.696005:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830194.696016:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3630 00000100:00080000:2.0:1787830194.696027:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830194.696035:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787830194.696073:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f480 x1874672086000128/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830194.696154:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672086000128:192.168.203.141@tcp:250:kworker.0 00000800:00000100:0.0:1787830194.696226:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1787830194.696256:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830194.696259:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787830194.696261:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830194.696265:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3630 00000100:00080000:2.0:1787830194.696269:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830194.696273:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830194.696283:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830194.696295:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c780 x1874672086000256/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830194.696349:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830194/real 1787830194] req@ffff9e35c498f480 x1874672086000128/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830219 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830194.696364:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f480 x1874672086000128/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830219 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830194.696372:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830194.696376:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830194.696379:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672086000128:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1787830194.696402:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086000256:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830194.696423:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830194.696435:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830194/real 1787830194] req@ffff9e35c498c780 x1874672086000256/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830219 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830194.696446:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c780 x1874672086000256/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830219 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830194.696453:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830194.696456:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830194.696459:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086000256:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1787830194.696555:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830194.696559:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787830194.696562:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830194.696566:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3630 00000100:00080000:2.0:1787830194.696570:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830194.696574:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830194.696579:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830194.696589:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c3c0 x1874672086000384/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830194.696631:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086000384:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830194.696652:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1787830194.696672:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830194.696676:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787830194.696677:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830194.696682:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3630 00000100:00080000:2.0:1787830194.696686:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830194.696689:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830194.696694:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:2.0:1787830194.696706:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f840 x1874672086000512/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830194.696735:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830194/real 1787830194] req@ffff9e35c498c3c0 x1874672086000384/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830219 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830194.696742:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c3c0 x1874672086000384/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830219 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830194.696747:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830194.696749:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830194.696751:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086000384:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830194.696760:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086000512:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830194.696774:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830194.696780:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830194/real 1787830194] req@ffff9e35c498f840 x1874672086000512/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830219 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830194.696829:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f840 x1874672086000512/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830219 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830194.696838:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830194.696841:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830194.696844:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086000512:192.168.203.141@tcp:8:kworker.0 00000100:00080000:2.0:1787830194.696948:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830194.696952:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787830194.696954:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830194.696959:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3630 00000100:00080000:2.0:1787830194.696963:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830194.696968:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830194.696974:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:2.0:1787830194.696987:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f0c0 x1874672086000640/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830194.697031:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086000640:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830194.697054:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1787830194.697075:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830194.697078:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787830194.697080:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830194.697084:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3630 00000100:00080000:2.0:1787830194.697088:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830194.697092:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830194.697097:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830194.697108:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cf00 x1874672086000768/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830194.697145:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830194/real 1787830194] req@ffff9e35c498f0c0 x1874672086000640/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830219 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830194.697157:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f0c0 x1874672086000640/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830219 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830194.697166:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830194.697169:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830194.697172:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086000640:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830194.697184:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086000768:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830194.697205:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830194.697216:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830194/real 1787830194] req@ffff9e35c498cf00 x1874672086000768/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830219 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830194.697227:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498cf00 x1874672086000768/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830219 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830194.697241:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830194.697244:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830194.697246:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086000768:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1787830194.697332:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830194.697336:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787830194.697337:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830194.697342:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3630 00000100:00080000:2.0:1787830194.697346:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830194.697349:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830194.697354:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830194.697368:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cf00 x1874672086000896/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830194.697409:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086000896:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830194.697431:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1787830194.697450:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830194.697454:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787830194.697456:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830194.697460:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3630 00000100:00080000:2.0:1787830194.697463:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830194.697466:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830194.697471:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:2.0:1787830194.697482:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f0c0 x1874672086001024/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830194.697518:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830194/real 1787830194] req@ffff9e35c498cf00 x1874672086000896/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830219 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830194.697530:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498cf00 x1874672086000896/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830219 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830194.697536:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830194.697539:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830194.697541:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086000896:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830194.697552:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086001024:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830194.697570:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830194.697580:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830194/real 1787830194] req@ffff9e35c498f0c0 x1874672086001024/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830219 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830194.697590:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f0c0 x1874672086001024/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830219 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830194.697599:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830194.697601:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830194.697605:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086001024:192.168.203.141@tcp:8:kworker.0 00000100:00080000:2.0:1787830194.697647:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830194.697650:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787830194.697652:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830194.697656:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3630 00000100:00080000:2.0:1787830194.697660:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830194.697663:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830194.697668:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:2.0:1787830194.697679:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f840 x1874672086001152/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830194.697739:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086001152:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830194.697759:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830194.697769:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830194/real 1787830194] req@ffff9e35c498f840 x1874672086001152/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830219 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830194.697780:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f840 x1874672086001152/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830219 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830194.697788:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830194.697791:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830194.697794:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086001152:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830194.824887:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830194.888019:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:0.0:1787830194.888738:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830194.888794:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787830195.848035:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830195.911982:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787830195.912107:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:0.0:1787830195.912235:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830196.872029:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830196.935933:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787830196.937046:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830196.937064:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787830197.896004:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830197.959947:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787830197.960056:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830197.960412:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787830198.920240:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830198.983991:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787830198.984061:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:0.0:1787830198.984183:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787830199.816052:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830199.816063:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787830199.816068:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830199.816079:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3635 00000100:00080000:2.0:1787830199.816089:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830199.816096:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787830199.816129:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c3c0 x1874672086001280/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830199.816209:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672086001280:192.168.203.141@tcp:250:kworker.0 00000800:00000100:0.0:1787830199.816266:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1787830199.816297:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830199.816301:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787830199.816303:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830199.816308:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3635 00000100:00080000:2.0:1787830199.816311:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830199.816315:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830199.816325:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830199.816337:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c780 x1874672086001408/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830199.816392:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830199/real 1787830199] req@ffff9e35c498c3c0 x1874672086001280/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830224 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830199.816413:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c3c0 x1874672086001280/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830224 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830199.816423:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830199.816427:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830199.816430:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672086001280:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1787830199.816452:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086001408:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830199.816474:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830199.816486:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830199/real 1787830199] req@ffff9e35c498c780 x1874672086001408/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830224 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830199.816497:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c780 x1874672086001408/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830224 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830199.816506:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830199.816509:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830199.816512:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086001408:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1787830199.816608:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830199.816613:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787830199.816615:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830199.816619:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3635 00000100:00080000:2.0:1787830199.816623:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830199.816627:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830199.816632:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830199.816645:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f480 x1874672086001536/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830199.816655:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830199.816657:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787830199.816659:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830199.816663:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3635 00000100:00080000:2.0:1787830199.816667:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830199.816670:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830199.816674:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:2.0:1787830199.816683:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e1c0 x1874672086001664/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830199.816691:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830199.816694:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787830199.816695:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830199.816699:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3635 00000100:00080000:2.0:1787830199.816701:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830199.816705:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830199.816708:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:2.0:1787830199.816717:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e580 x1874672086001792/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830199.816725:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830199.816728:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787830199.816730:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830199.816734:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3635 00000100:00080000:2.0:1787830199.816737:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830199.816740:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830199.816744:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830199.816753:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498fc00 x1874672086001920/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830199.816761:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830199.816764:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787830199.816765:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830199.816769:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3635 00000100:00080000:2.0:1787830199.816772:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830199.816775:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830199.816779:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830199.816788:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498ed00 x1874672086002048/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830199.816796:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830199.817069:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787830199.817071:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830199.817076:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3635 00000100:00080000:2.0:1787830199.817080:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830199.817084:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830199.817089:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:2.0:1787830199.817101:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c000 x1874672086002176/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830199.817111:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830199.817114:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787830199.817116:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830199.817119:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3635 00000100:00080000:2.0:1787830199.817122:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830199.817125:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830199.817129:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:2.0:1787830199.817136:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498de00 x1874672086002304/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830199.817197:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086001536:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830199.817230:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830199.817242:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830199/real 1787830199] req@ffff9e35c498f480 x1874672086001536/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830224 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830199.817254:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f480 x1874672086001536/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830224 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830199.817267:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830199.817270:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830199.817274:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086001536:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830199.817280:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086001664:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830199.817297:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830199.817307:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830199/real 1787830199] req@ffff9e35c498e1c0 x1874672086001664/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830224 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830199.817318:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e1c0 x1874672086001664/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830224 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830199.817326:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830199.817329:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830199.817332:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086001664:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830199.817338:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086001792:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830199.817352:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830199.817360:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830199/real 1787830199] req@ffff9e35c498e580 x1874672086001792/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830224 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830199.817371:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e580 x1874672086001792/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830224 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830199.817379:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830199.817382:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830199.817385:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086001792:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830199.817390:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086001920:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830199.817405:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830199.817415:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830199/real 1787830199] req@ffff9e35c498fc00 x1874672086001920/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830224 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830199.817424:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498fc00 x1874672086001920/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830224 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830199.817432:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830199.817435:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830199.817439:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086001920:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830199.817444:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086002048:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830199.817459:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830199.817467:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830199/real 1787830199] req@ffff9e35c498ed00 x1874672086002048/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830224 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830199.817478:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498ed00 x1874672086002048/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830224 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830199.817485:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830199.817488:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830199.817491:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086002048:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830199.817496:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086002176:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830199.817512:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830199.817521:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830199/real 1787830199] req@ffff9e35c498c000 x1874672086002176/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830224 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830199.817532:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c000 x1874672086002176/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830224 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830199.817540:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830199.817543:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830199.817546:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086002176:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830199.817551:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086002304:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830199.817563:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830199.817571:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830199/real 1787830199] req@ffff9e35c498de00 x1874672086002304/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830224 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830199.817581:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498de00 x1874672086002304/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830224 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830199.817589:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830199.817592:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830199.817595:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086002304:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830199.943942:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787830200.007937:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787830200.008027:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830200.008028:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787830200.967957:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830201.031934:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787830201.034063:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830201.034139:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787830201.992078:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830202.055898:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787830202.055899:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787830202.055952:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830203.016049:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830203.079934:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787830203.079989:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830203.080049:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787830204.040013:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830204.103892:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787830204.104924:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787830204.105126:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787830204.935990:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830204.936001:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787830204.936004:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830204.936016:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3640 00000100:00080000:3.0:1787830204.936027:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830204.936034:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787830204.936070:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498f00 x1874672086002432/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830204.936165:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672086002432:192.168.203.141@tcp:250:kworker.0 00000800:00000100:0.0:1787830204.936235:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1787830204.936266:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830204.936270:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787830204.936272:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830204.936276:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3640 00000100:00080000:3.0:1787830204.936281:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830204.936294:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830204.936303:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830204.936317:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498b40 x1874672086002560/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830204.936540:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830204/real 1787830204] req@ffff9e35c6498f00 x1874672086002432/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830229 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830204.936558:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498f00 x1874672086002432/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830229 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830204.936568:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830204.936572:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830204.936576:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672086002432:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1787830204.936602:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086002560:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830204.936625:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830204.936650:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830204/real 1787830204] req@ffff9e35c6498b40 x1874672086002560/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830229 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830204.936662:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498b40 x1874672086002560/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830229 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830204.936671:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830204.936675:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830204.936678:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086002560:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1787830204.936860:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830204.936864:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787830204.936865:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830204.936868:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3640 00000100:00080000:3.0:1787830204.936872:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830204.936874:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830204.936879:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830204.936890:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499a40 x1874672086002688/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830204.937006:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086002688:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830204.937030:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1787830204.937052:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830204.937055:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787830204.937057:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830204.937060:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3640 00000100:00080000:3.0:1787830204.937063:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830204.937066:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830204.937070:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:0.0:1787830204.937142:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830204/real 1787830204] req@ffff9e35c6499a40 x1874672086002688/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830229 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830204.937154:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499a40 x1874672086002688/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830229 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830204.937161:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830204.937165:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830204.937167:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086002688:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787830204.937200:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499680 x1874672086002816/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830204.937217:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830204.937222:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787830204.937224:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830204.937228:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3640 00000100:00080000:3.0:1787830204.937232:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830204.937236:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830204.937241:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:3.0:1787830204.937254:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a940 x1874672086002944/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830204.937262:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830204.937265:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787830204.937267:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830204.937270:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3640 00000100:00080000:3.0:1787830204.937274:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830204.937277:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830204.937281:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830204.937291:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b0c0 x1874672086003072/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830204.937298:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830204.937301:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787830204.937303:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830204.937306:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3640 00000100:00080000:3.0:1787830204.937310:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830204.937313:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830204.937317:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830204.937326:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a580 x1874672086003200/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830204.937334:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830204.937337:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787830204.937339:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830204.937342:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3640 00000100:00100000:0.0:1787830204.937456:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086002816:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830204.937479:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830204.937490:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830204/real 1787830204] req@ffff9e35c6499680 x1874672086002816/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830229 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830204.937500:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499680 x1874672086002816/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830229 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830204.937508:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830204.937511:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830204.937514:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086002816:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830204.937519:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086002944:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830204.937533:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830204.937542:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830204/real 1787830204] req@ffff9e35c649a940 x1874672086002944/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830229 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830204.937551:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a940 x1874672086002944/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830229 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830204.937558:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830204.937561:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830204.937564:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086002944:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830204.937568:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086003072:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830204.937584:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830204.937592:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830204/real 1787830204] req@ffff9e35c649b0c0 x1874672086003072/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830229 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830204.937602:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b0c0 x1874672086003072/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830229 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830204.937609:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830204.937612:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830204.937614:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086003072:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830204.937620:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086003200:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1787830204.937754:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830204.937760:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830204.937765:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:3.0:1787830204.937778:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499e00 x1874672086003328/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830204.937788:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830204.937791:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787830204.937793:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830204.937796:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3640 00000100:00080000:3.0:1787830204.937819:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830204.937823:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830204.937828:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000800:00000100:0.0:1787830204.937987:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830204.938001:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830204/real 1787830204] req@ffff9e35c649a580 x1874672086003200/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830229 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830204.938013:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a580 x1874672086003200/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830229 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830204.938020:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830204.938023:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830204.938027:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086003200:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830204.938046:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086003328:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830204.938066:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830204.938076:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830204/real 1787830204] req@ffff9e35c6499e00 x1874672086003328/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830229 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787830204.938131:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499a40 x1874672086003456/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830204.938241:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499e00 x1874672086003328/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830229 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830204.938250:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830204.938253:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830204.938256:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086003328:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830204.938267:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086003456:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830204.938288:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830204.938298:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830204/real 1787830204] req@ffff9e35c6499a40 x1874672086003456/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830229 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830204.938309:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499a40 x1874672086003456/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830229 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830204.938318:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830204.938321:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830204.938324:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086003456:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830205.065006:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830205.127928:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787830205.128019:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:0.0:1787830205.128048:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830206.088120:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830206.151943:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787830206.151943:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787830206.151996:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830207.111994:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830207.176038:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787830207.176638:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830207.176658:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787830208.136059:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:0.0:1787830208.200110:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830208.200204:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787830208.200267:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787830209.159917:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787830209.223944:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787830209.225072:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830209.225900:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00080000:2.0:1787830210.056108:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830210.056119:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787830210.056123:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830210.056134:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3645 00000100:00080000:2.0:1787830210.056144:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830210.056151:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787830210.056185:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e940 x1874672086003584/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830210.056220:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830210.056224:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787830210.056226:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830210.056231:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3645 00000100:00080000:2.0:1787830210.056235:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830210.056239:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787830210.056253:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672086003584:192.168.203.141@tcp:250:kworker.0 00000008:00100000:2.0:1787830210.056258:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830210.056275:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498d680 x1874672086003712/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830210.056286:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830210.056290:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787830210.056291:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830210.056296:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3645 00000100:00080000:2.0:1787830210.056299:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830210.056303:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:0.0:1787830210.056306:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:2.0:1787830210.056311:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830210.056326:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cb40 x1874672086003840/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830210.056336:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830210.056339:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:0.0:1787830210.056341:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830210/real 1787830210] req@ffff9e35c498e940 x1874672086003584/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830235 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830210.056342:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830210.056351:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3645 00000100:00080000:2.0:1787830210.056357:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787830210.056360:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e940 x1874672086003584/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830235 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830210.056364:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830210.056374:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00080000:0.0:1787830210.056376:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830210.056382:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830210.056384:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672086003584:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1787830210.056391:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498de00 x1874672086003968/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830210.056404:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787830210.056405:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086003712:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1787830210.056413:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787830210.056416:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830210.056422:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3645 00000800:00000100:0.0:1787830210.056426:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1787830210.056426:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830210.056445:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787830210.056448:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830210/real 1787830210] req@ffff9e35c498d680 x1874672086003712/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830235 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1787830210.056452:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:0.0:1787830210.056462:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498d680 x1874672086003712/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830235 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830210.056469:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:2.0:1787830210.056470:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c000 x1874672086004096/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830210.056475:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830210.056481:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086003712:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830210.056491:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086003840:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1787830210.056491:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830210.056498:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787830210.056500:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830210.056506:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3645 00000800:00000100:0.0:1787830210.056509:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1787830210.056513:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787830210.056518:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830210/real 1787830210] req@ffff9e35c498cb40 x1874672086003840/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830235 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830210.056520:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830210.056528:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830210.056532:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498cb40 x1874672086003840/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830235 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830210.056539:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830210.056544:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787830210.056545:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498ed00 x1874672086004224/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830210.056550:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086003840:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1787830210.056562:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830210.056565:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787830210.056567:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830210.056572:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3645 00000100:00100000:0.0:1787830210.056575:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086003968:192.168.203.141@tcp:8:kworker.0 00000100:00080000:2.0:1787830210.056577:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830210.056585:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830210.056591:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1787830210.056603:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787830210.056606:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498fc00 x1874672086004352/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830210.056617:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830210/real 1787830210] req@ffff9e35c498de00 x1874672086003968/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830235 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830210.056618:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830210.056626:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787830210.056631:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787830210.056635:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498de00 x1874672086003968/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830235 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830210.056639:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3645 00000100:00080000:2.0:1787830210.056646:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830210.056648:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830210.056653:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:2.0:1787830210.056656:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787830210.056661:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086003968:192.168.203.141@tcp:8:kworker.0 00000008:00100000:2.0:1787830210.056663:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:0.0:1787830210.056670:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086004096:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787830210.056678:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e580 x1874672086004480/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830210.056688:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830210.056691:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787830210.056693:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1787830210.056696:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1787830210.056700:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3645 00000100:00080000:2.0:1787830210.056706:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830210.056711:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787830210.056714:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830210/real 1787830210] req@ffff9e35c498c000 x1874672086004096/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830235 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830210.056727:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c000 x1874672086004096/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830235 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830210.056734:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830210.056737:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830210.056739:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086004096:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830210.056745:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086004224:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830210.056761:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830210.056770:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830210/real 1787830210] req@ffff9e35c498ed00 x1874672086004224/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830235 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830210.056780:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498ed00 x1874672086004224/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830235 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830210.056786:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830210.056789:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830210.056791:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086004224:192.168.203.141@tcp:38:kworker.0 00000008:00100000:2.0:1787830210.056968:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:2.0:1787830210.056985:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e1c0 x1874672086004608/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830210.057093:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086004352:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830210.057117:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830210.057127:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830210/real 1787830210] req@ffff9e35c498fc00 x1874672086004352/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830235 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830210.057136:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498fc00 x1874672086004352/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830235 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830210.057143:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830210.057146:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830210.057148:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086004352:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830210.057153:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086004480:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830210.057167:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830210.057176:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830210/real 1787830210] req@ffff9e35c498e580 x1874672086004480/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830235 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830210.057186:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e580 x1874672086004480/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830235 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830210.057194:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830210.057196:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830210.057199:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086004480:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830210.057212:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086004608:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830210.057228:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830210.057237:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830210/real 1787830210] req@ffff9e35c498e1c0 x1874672086004608/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830235 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830210.057247:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e1c0 x1874672086004608/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830235 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830210.057254:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830210.057257:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830210.057260:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086004608:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830210.184359:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830210.247945:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787830210.248856:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830210.248963:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787830211.208114:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830211.271992:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787830211.272238:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787830211.272838:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830212.231936:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787830212.296111:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830212.296439:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787830212.297244:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787830213.255950:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830213.319931:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787830213.319931:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787830213.320326:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830214.279961:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830214.343936:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787830214.344083:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787830214.344084:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787830215.177740:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830215.177747:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787830215.177751:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830215.177762:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3650 00000100:00080000:3.0:1787830215.177781:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830215.177789:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787830215.177844:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498b40 x1874672086004736/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830215.177879:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830215.177884:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787830215.177886:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830215.177890:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3650 00000100:00080000:3.0:1787830215.177893:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830215.177897:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830215.177907:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830215.177919:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498f00 x1874672086004864/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830215.177928:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830215.177931:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787830215.177932:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830215.177936:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3650 00000100:00080000:3.0:1787830215.177939:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830215.177942:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830215.177947:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830215.177955:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498780 x1874672086004992/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830215.177963:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830215.177966:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787830215.177968:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830215.177971:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3650 00000100:00080000:3.0:1787830215.177975:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830215.177978:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830215.177982:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:3.0:1787830215.177991:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c64983c0 x1874672086005120/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830215.178000:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830215.178002:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787830215.178004:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830215.178007:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3650 00000100:00080000:3.0:1787830215.178010:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830215.178013:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830215.178017:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:3.0:1787830215.178026:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649ad00 x1874672086005248/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830215.178034:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830215.178038:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787830215.178039:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830215.178043:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3650 00000100:00080000:3.0:1787830215.178046:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830215.178049:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830215.178052:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830215.178060:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c64992c0 x1874672086005376/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830215.178072:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830215.178075:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787830215.178076:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830215.178079:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3650 00000100:00080000:3.0:1787830215.178082:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830215.178085:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830215.178089:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830215.178098:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b480 x1874672086005504/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830215.178105:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830215.178108:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787830215.178109:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830215.178113:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3650 00000100:00080000:3.0:1787830215.178116:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830215.178119:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830215.178123:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:3.0:1787830215.178133:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649bc00 x1874672086005632/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830215.178140:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830215.178143:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787830215.178145:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830215.178149:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3650 00000100:00080000:3.0:1787830215.178152:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830215.178155:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830215.178159:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:3.0:1787830215.178167:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b840 x1874672086005760/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787830215.179127:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672086004736:192.168.203.141@tcp:250:kworker.0 00000800:00000100:1.0:1787830215.179192:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830215.179231:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830215/real 1787830215] req@ffff9e35c6498b40 x1874672086004736/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830240 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830215.179247:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498b40 x1874672086004736/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830240 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830215.179256:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830215.179260:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830215.179264:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672086004736:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1787830215.179270:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086004864:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787830215.179290:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830215.179302:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830215/real 1787830215] req@ffff9e35c6498f00 x1874672086004864/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830240 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830215.179313:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498f00 x1874672086004864/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830240 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830215.179321:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830215.179324:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830215.179327:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086004864:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787830215.179334:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086004992:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787830215.179351:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830215.179360:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830215/real 1787830215] req@ffff9e35c6498780 x1874672086004992/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830240 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830215.179371:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498780 x1874672086004992/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830240 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830215.179377:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830215.179380:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830215.179383:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086004992:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787830215.179388:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c64983c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086005120:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787830215.179406:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830215.179417:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830215/real 1787830215] req@ffff9e35c64983c0 x1874672086005120/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830240 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830215.179428:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c64983c0 x1874672086005120/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830240 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830215.179435:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830215.179438:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830215.179440:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c64983c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086005120:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830215.179446:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086005248:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787830215.179463:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830215.179472:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830215/real 1787830215] req@ffff9e35c649ad00 x1874672086005248/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830240 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830215.179484:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649ad00 x1874672086005248/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830240 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830215.179491:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830215.179494:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830215.179496:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086005248:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830215.179502:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c64992c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086005376:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787830215.179519:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830215.179528:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830215/real 1787830215] req@ffff9e35c64992c0 x1874672086005376/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830240 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830215.179538:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c64992c0 x1874672086005376/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830240 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830215.179546:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830215.179549:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830215.179552:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c64992c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086005376:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787830215.179557:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086005504:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787830215.179573:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830215.179582:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830215/real 1787830215] req@ffff9e35c649b480 x1874672086005504/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830240 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830215.179591:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b480 x1874672086005504/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830240 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830215.179602:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830215.179605:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830215.179608:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086005504:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787830215.179613:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086005632:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787830215.179628:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830215.179635:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830215/real 1787830215] req@ffff9e35c649bc00 x1874672086005632/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830240 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830215.179644:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649bc00 x1874672086005632/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830240 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830215.179652:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830215.179655:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830215.179657:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086005632:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830215.179663:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086005760:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787830215.179747:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830215.179759:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830215/real 1787830215] req@ffff9e35c649b840 x1874672086005760/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830240 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830215.179780:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b840 x1874672086005760/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830240 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830215.179788:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830215.179791:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830215.179794:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086005760:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830215.303943:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:3.0:1787830215.367971:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787830215.367972:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787830215.368486:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830216.328107:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830216.392046:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787830216.392153:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787830216.392157:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830217.352195:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830217.415959:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787830217.416622:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830217.417294:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787830218.376139:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830218.439940:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787830218.441551:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787830218.441850:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830219.400056:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830219.463869:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787830219.463891:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787830219.463984:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787830220.295956:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830220.295969:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787830220.295973:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830220.295984:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3655 00000100:00080000:3.0:1787830220.295995:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830220.296002:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787830220.296038:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498000 x1874672086005888/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830220.296072:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830220.296076:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787830220.296077:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830220.296081:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3655 00000100:00080000:3.0:1787830220.296084:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830220.296088:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830220.296098:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830220.296109:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a1c0 x1874672086006016/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830220.296119:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830220.296122:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787830220.296124:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830220.296127:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3655 00000100:00080000:3.0:1787830220.296130:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830220.296134:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830220.296138:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830220.296150:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b840 x1874672086006144/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830220.296159:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830220.296162:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787830220.296163:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830220.296167:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3655 00000100:00080000:3.0:1787830220.296170:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830220.296173:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830220.296178:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:3.0:1787830220.296186:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649bc00 x1874672086006272/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830220.296195:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830220.296198:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787830220.296199:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830220.296203:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3655 00000100:00080000:3.0:1787830220.296206:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830220.296209:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830220.296213:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:3.0:1787830220.296222:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b480 x1874672086006400/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830220.296230:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830220.296233:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787830220.296235:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830220.296239:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3655 00000100:00080000:3.0:1787830220.296242:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830220.296245:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830220.296249:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830220.296259:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c64992c0 x1874672086006528/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830220.296266:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830220.296269:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787830220.296271:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830220.296274:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3655 00000100:00080000:3.0:1787830220.296277:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830220.296280:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830220.296284:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830220.296294:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649ad00 x1874672086006656/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830220.296307:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830220.296310:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787830220.296312:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830220.296315:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3655 00000100:00080000:3.0:1787830220.296319:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830220.296322:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830220.296326:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:3.0:1787830220.296335:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c64983c0 x1874672086006784/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830220.296342:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830220.296344:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787830220.296346:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830220.296349:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3655 00000100:00080000:3.0:1787830220.296352:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830220.296355:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830220.296371:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:3.0:1787830220.296382:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498780 x1874672086006912/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787830220.296509:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672086005888:192.168.203.141@tcp:250:kworker.0 00000800:00000100:1.0:1787830220.296577:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830220.296615:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830220/real 1787830220] req@ffff9e35c6498000 x1874672086005888/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830245 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830220.296630:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498000 x1874672086005888/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830245 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830220.296640:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830220.296643:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830220.296647:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672086005888:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1787830220.296653:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086006016:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787830220.296672:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830220.296683:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830220/real 1787830220] req@ffff9e35c649a1c0 x1874672086006016/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830245 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830220.296694:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a1c0 x1874672086006016/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830245 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830220.296701:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830220.296704:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830220.296707:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086006016:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787830220.296712:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086006144:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787830220.296728:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830220.296752:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830220/real 1787830220] req@ffff9e35c649b840 x1874672086006144/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830245 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830220.296763:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b840 x1874672086006144/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830245 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830220.296770:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830220.296773:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830220.296776:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086006144:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787830220.296781:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086006272:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787830220.296820:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830220.296843:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830220/real 1787830220] req@ffff9e35c649bc00 x1874672086006272/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830245 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830220.296855:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649bc00 x1874672086006272/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830245 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830220.296862:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830220.296865:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830220.297012:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086006272:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830220.297020:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086006400:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787830220.297042:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830220.297053:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830220/real 1787830220] req@ffff9e35c649b480 x1874672086006400/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830245 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830220.297071:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b480 x1874672086006400/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830245 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830220.297080:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830220.297083:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830220.297086:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086006400:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830220.297091:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c64992c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086006528:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787830220.297108:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830220.297118:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830220/real 1787830220] req@ffff9e35c64992c0 x1874672086006528/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830245 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830220.297129:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c64992c0 x1874672086006528/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830245 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830220.297136:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830220.297139:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830220.297142:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c64992c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086006528:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787830220.297147:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086006656:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787830220.297164:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830220.297174:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830220/real 1787830220] req@ffff9e35c649ad00 x1874672086006656/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830245 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830220.297183:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649ad00 x1874672086006656/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830245 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830220.297196:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830220.297198:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830220.297201:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086006656:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787830220.297207:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c64983c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086006784:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787830220.297222:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830220.297230:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830220/real 1787830220] req@ffff9e35c64983c0 x1874672086006784/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830245 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830220.297240:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c64983c0 x1874672086006784/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830245 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830220.297248:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830220.297251:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830220.297253:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c64983c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086006784:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830220.297256:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086006912:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787830220.297268:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830220.297276:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830220/real 1787830220] req@ffff9e35c6498780 x1874672086006912/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830245 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830220.297287:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498780 x1874672086006912/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830245 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830220.297293:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830220.297295:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830220.297298:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086006912:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830220.423926:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830220.487933:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787830220.487955:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787830220.488187:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830221.448226:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830221.511921:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787830221.511921:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787830221.512165:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830222.471897:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787830222.536119:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830222.536221:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787830222.536515:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787830223.495947:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830223.559960:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787830223.560394:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787830223.560604:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830224.519957:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830224.583887:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787830224.583964:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787830224.586731:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787830225.415931:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830225.415942:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787830225.415948:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830225.415961:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3660 00000100:00080000:3.0:1787830225.415974:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830225.415983:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787830225.416032:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498f00 x1874672086007040/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830225.416069:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830225.416073:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787830225.416074:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830225.416079:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3660 00000100:00080000:3.0:1787830225.416082:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830225.416085:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830225.416097:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830225.416109:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498b40 x1874672086007168/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830225.416118:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830225.416121:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787830225.416122:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830225.416126:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3660 00000100:00080000:3.0:1787830225.416128:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830225.416131:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830225.416135:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830225.416143:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499a40 x1874672086007296/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830225.416150:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830225.416154:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787830225.416155:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830225.416159:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3660 00000100:00080000:3.0:1787830225.416161:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830225.416164:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830225.416168:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:3.0:1787830225.416176:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499e00 x1874672086007424/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830225.416183:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830225.416186:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787830225.416188:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830225.416191:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3660 00000100:00080000:3.0:1787830225.416194:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830225.416196:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830225.416200:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:3.0:1787830225.416223:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a580 x1874672086007552/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830225.416232:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830225.416235:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787830225.416236:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830225.416240:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3660 00000100:00080000:3.0:1787830225.416243:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830225.416246:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830225.416251:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830225.416259:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b0c0 x1874672086007680/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830225.416267:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830225.416270:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787830225.416272:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830225.416275:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3660 00000100:00080000:3.0:1787830225.416277:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830225.416280:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830225.416284:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830225.416292:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a940 x1874672086007808/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830225.416299:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830225.416303:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787830225.416304:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830225.416307:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3660 00000100:00080000:3.0:1787830225.416309:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830225.416313:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830225.416317:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:3.0:1787830225.416325:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499680 x1874672086007936/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830225.416332:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830225.416335:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787830225.416337:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830225.416341:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3660 00000100:00080000:3.0:1787830225.416343:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830225.416346:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830225.416350:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:3.0:1787830225.416358:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498780 x1874672086008064/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787830225.416498:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672086007040:192.168.203.141@tcp:250:kworker.0 00000800:00000100:1.0:1787830225.416571:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830225.416613:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830225/real 1787830225] req@ffff9e35c6498f00 x1874672086007040/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830250 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830225.416631:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498f00 x1874672086007040/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830250 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830225.416641:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830225.416645:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830225.416649:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672086007040:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1787830225.416655:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086007168:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787830225.416675:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830225.416686:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830225/real 1787830225] req@ffff9e35c6498b40 x1874672086007168/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830250 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830225.416709:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498b40 x1874672086007168/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830250 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830225.416724:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830225.416728:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830225.416731:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086007168:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787830225.416737:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086007296:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787830225.416754:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830225.416763:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830225/real 1787830225] req@ffff9e35c6499a40 x1874672086007296/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830250 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830225.416772:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499a40 x1874672086007296/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830250 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830225.416778:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830225.416782:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830225.416784:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086007296:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787830225.416790:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086007424:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787830225.416838:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830225.416854:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830225/real 1787830225] req@ffff9e35c6499e00 x1874672086007424/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830250 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830225.416865:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499e00 x1874672086007424/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830250 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830225.416873:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830225.416875:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830225.416879:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086007424:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830225.416883:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086007552:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787830225.416901:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830225.416911:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830225/real 1787830225] req@ffff9e35c649a580 x1874672086007552/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830250 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830225.416921:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a580 x1874672086007552/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830250 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830225.416929:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830225.416931:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830225.416934:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086007552:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830225.416940:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086007680:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787830225.416957:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830225.416967:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830225/real 1787830225] req@ffff9e35c649b0c0 x1874672086007680/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830250 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830225.416978:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b0c0 x1874672086007680/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830250 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830225.416986:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830225.416989:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830225.416992:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086007680:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787830225.416998:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086007808:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787830225.417015:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830225.417025:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830225/real 1787830225] req@ffff9e35c649a940 x1874672086007808/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830250 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830225.417035:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a940 x1874672086007808/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830250 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830225.417043:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830225.417046:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830225.417049:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086007808:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787830225.417054:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086007936:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787830225.417071:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830225.417081:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830225/real 1787830225] req@ffff9e35c6499680 x1874672086007936/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830250 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830225.417091:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499680 x1874672086007936/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830250 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830225.417099:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830225.417102:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830225.417104:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086007936:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830225.417109:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086008064:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787830225.417126:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830225.417135:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830225/real 1787830225] req@ffff9e35c6498780 x1874672086008064/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830250 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830225.417144:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498780 x1874672086008064/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830250 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830225.417153:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830225.417156:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830225.417159:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086008064:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830225.543966:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830225.607924:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787830225.608054:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787830225.609244:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830226.567936:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:3.0:1787830226.632096:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787830226.632165:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830226.632251:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787830227.593749:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830227.655922:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787830227.655970:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787830227.657496:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830228.615934:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830228.679921:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787830228.679965:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787830228.679973:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830229.640056:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787830229.704171:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830229.704328:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787830229.704430:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00080000:2.0:1787830230.538629:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830230.538643:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787830230.538648:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830230.538659:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3665 00000100:00080000:2.0:1787830230.538681:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830230.538690:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787830230.538737:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f480 x1874672086008192/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830230.538769:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830230.538777:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787830230.538779:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830230.538783:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3665 00000100:00080000:2.0:1787830230.538787:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830230.538791:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830230.538826:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830230.538840:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c780 x1874672086008320/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830230.538851:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830230.538855:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787830230.538857:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830230.538862:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3665 00000100:00080000:2.0:1787830230.538865:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830230.538869:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787830230.538876:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672086008192:192.168.203.141@tcp:250:kworker.0 00000008:00100000:2.0:1787830230.538877:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830230.538891:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c3c0 x1874672086008448/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830230.538901:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830230.538905:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787830230.538906:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830230.538910:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3665 00000100:00080000:2.0:1787830230.538914:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830230.538918:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830230.538924:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:2.0:1787830230.538936:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f840 x1874672086008576/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830230.538945:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830230.538949:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787830230.538951:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830230.538955:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3665 00000100:00080000:2.0:1787830230.538959:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830230.538965:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:1.0:1787830230.538966:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:2.0:1787830230.538971:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:2.0:1787830230.538986:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f0c0 x1874672086008704/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830230.538996:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830230.538999:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787830230.539000:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830230.539006:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3665 00000100:00080000:2.0:1787830230.539010:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830230.539014:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830230.539020:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830230.539025:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830230/real 1787830230] req@ffff9e35c498f480 x1874672086008192/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830255 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830230.539036:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cf00 x1874672086008832/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830230.539047:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787830230.539051:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f480 x1874672086008192/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830255 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830230.539052:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787830230.539056:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830230.539064:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3665 00000100:00080000:2.0:1787830230.539070:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830230.539072:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830230.539075:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:1.0:1787830230.539081:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000008:00100000:2.0:1787830230.539083:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830230.539097:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e1c0 x1874672086008960/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830230.539106:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830230.539110:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787830230.539112:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830230.539116:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3665 00000100:00080000:2.0:1787830230.539119:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830230.539124:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830230.539129:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:2.0:1787830230.539140:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e580 x1874672086009088/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830230.539148:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830230.539151:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787830230.539152:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787830230.539156:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672086008192:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1787830230.539157:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3665 00000100:00080000:2.0:1787830230.539163:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830230.539168:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787830230.539172:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086008320:192.168.203.141@tcp:38:kworker.0 00000008:00100000:2.0:1787830230.539175:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:2.0:1787830230.539190:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498fc00 x1874672086009216/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787830230.539208:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830230.539224:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830230/real 1787830230] req@ffff9e35c498c780 x1874672086008320/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830255 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830230.539238:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c780 x1874672086008320/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830255 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830230.539248:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830230.539251:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830230.539255:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086008320:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787830230.539352:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086008448:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787830230.539378:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830230.539459:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830230/real 1787830230] req@ffff9e35c498c3c0 x1874672086008448/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830255 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830230.539473:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c3c0 x1874672086008448/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830255 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830230.539481:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830230.539485:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830230.539488:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086008448:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787830230.539495:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086008576:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787830230.539516:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830230.539528:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830230/real 1787830230] req@ffff9e35c498f840 x1874672086008576/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830255 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830230.539540:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f840 x1874672086008576/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830255 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830230.539555:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830230.539559:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830230.539561:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086008576:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830230.539567:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086008704:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787830230.539587:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830230.539598:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830230/real 1787830230] req@ffff9e35c498f0c0 x1874672086008704/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830255 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830230.539608:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f0c0 x1874672086008704/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830255 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830230.539616:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830230.539619:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830230.539623:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086008704:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830230.539628:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086008832:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787830230.539647:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830230.539657:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830230/real 1787830230] req@ffff9e35c498cf00 x1874672086008832/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830255 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830230.539680:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498cf00 x1874672086008832/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830255 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830230.539689:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830230.539692:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830230.539696:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086008832:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787830230.539701:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086008960:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787830230.539723:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830230.539734:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830230/real 1787830230] req@ffff9e35c498e1c0 x1874672086008960/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830255 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830230.539746:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e1c0 x1874672086008960/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830255 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830230.539754:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830230.539757:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830230.539761:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086008960:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787830230.539767:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086009088:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787830230.540521:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830230.540540:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830230/real 1787830230] req@ffff9e35c498e580 x1874672086009088/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830255 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830230.540552:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e580 x1874672086009088/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830255 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830230.540561:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830230.540564:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830230.540568:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086009088:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830230.540575:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086009216:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787830230.540597:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830230.540608:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830230/real 1787830230] req@ffff9e35c498fc00 x1874672086009216/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830255 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830230.540620:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498fc00 x1874672086009216/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830255 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830230.540628:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830230.540631:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830230.540635:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086009216:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830230.664728:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830230.727910:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787830230.727955:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787830230.728304:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830231.687891:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830231.751930:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787830231.752370:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787830231.754382:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830232.711937:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830232.775969:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787830232.776216:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830232.779732:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787830233.735935:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830233.799930:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787830233.799930:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787830233.799981:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830234.759905:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830234.823930:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787830234.823930:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787830234.824156:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787830235.655959:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830235.655970:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787830235.655974:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830235.655985:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3671 00000100:00080000:3.0:1787830235.655996:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830235.656003:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787830235.656039:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c64983c0 x1874672086009344/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830235.656066:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830235.656070:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787830235.656072:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830235.656076:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3671 00000100:00080000:3.0:1787830235.656079:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830235.656082:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830235.656103:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830235.656109:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c64983c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672086009344:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1787830235.656119:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649ad00 x1874672086009472/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830235.656131:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830235.656135:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787830235.656136:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830235.656140:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3671 00000100:00080000:3.0:1787830235.656144:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830235.656148:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830235.656153:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830235.656163:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c64992c0 x1874672086009600/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787830235.656172:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1787830235.656175:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830235.656179:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787830235.656181:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830235.656186:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3671 00000100:00080000:3.0:1787830235.656190:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830235.656194:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830235.656198:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:3.0:1787830235.656209:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b480 x1874672086009728/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787830235.656217:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830235/real 1787830235] req@ffff9e35c64983c0 x1874672086009344/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830260 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830235.656220:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830235.656226:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787830235.656230:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830235.656237:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3671 00000100:00080000:3.0:1787830235.656241:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830235.656245:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787830235.656248:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c64983c0 x1874672086009344/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830260 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1787830235.656250:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00080000:1.0:1787830235.656264:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1787830235.656267:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649bc00 x1874672086009856/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830235.656271:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830235.656280:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c64983c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672086009344:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1787830235.656285:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830235.656289:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787830235.656291:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830235.656296:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3671 00000100:00080000:3.0:1787830235.656299:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830235.656302:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830235.656307:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830235.656314:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086009472:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787830235.656319:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b840 x1874672086009984/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830235.656330:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830235.656333:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787830235.656335:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830235.656339:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3671 00000800:00000100:1.0:1787830235.656343:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1787830235.656344:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830235.656351:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787830235.656356:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830235/real 1787830235] req@ffff9e35c649ad00 x1874672086009472/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830260 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1787830235.656356:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830235.656369:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649ad00 x1874672086009472/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830260 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787830235.656372:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a1c0 x1874672086010112/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830235.656378:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830235.656385:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830235.656390:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086009472:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1787830235.656391:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830235.656396:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:1.0:1787830235.656397:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c64992c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086009600:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1787830235.656400:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830235.656407:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3671 00000100:00080000:3.0:1787830235.656411:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1787830235.656413:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1787830235.656418:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787830235.656423:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830235/real 1787830235] req@ffff9e35c64992c0 x1874672086009600/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830260 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1787830235.656425:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:1.0:1787830235.656436:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c64992c0 x1874672086009600/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830260 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830235.656441:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1787830235.656443:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498000 x1874672086010240/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830235.656446:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830235.656452:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c64992c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086009600:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787830235.656461:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086009728:192.168.203.141@tcp:8:kworker.0 00000100:00080000:3.0:1787830235.656465:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830235.656470:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787830235.656472:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1787830235.656476:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1787830235.656478:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3671 00000100:00080000:3.0:1787830235.656484:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1787830235.656485:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830235/real 1787830235] req@ffff9e35c649b480 x1874672086009728/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830260 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830235.656492:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787830235.656498:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b480 x1874672086009728/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830260 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1787830235.656499:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00080000:1.0:1787830235.656509:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830235.656512:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830235.656518:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086009728:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787830235.656518:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c64983c0 x1874672086010368/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787830235.656524:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086009856:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787830235.656539:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830235.656545:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830235/real 1787830235] req@ffff9e35c649bc00 x1874672086009856/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830260 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830235.656564:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649bc00 x1874672086009856/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830260 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830235.656575:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830235.656579:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830235.656582:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086009856:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830235.656599:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086009984:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787830235.656617:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830235.656640:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830235/real 1787830235] req@ffff9e35c649b840 x1874672086009984/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830260 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830235.656651:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b840 x1874672086009984/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830260 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830235.656658:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830235.656661:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830235.656663:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086009984:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787830235.656668:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086010112:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787830235.656684:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830235.656693:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830235/real 1787830235] req@ffff9e35c649a1c0 x1874672086010112/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830260 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830235.656702:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a1c0 x1874672086010112/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830260 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830235.656708:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830235.656711:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830235.656714:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086010112:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787830235.656719:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086010240:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787830235.656735:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830235.656744:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830235/real 1787830235] req@ffff9e35c6498000 x1874672086010240/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830260 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830235.656755:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498000 x1874672086010240/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830260 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830235.656762:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830235.656765:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830235.656768:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086010240:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830235.656774:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c64983c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086010368:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787830235.656791:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830235.656826:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830235/real 1787830235] req@ffff9e35c64983c0 x1874672086010368/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830260 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830235.656837:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c64983c0 x1874672086010368/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830260 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830235.656846:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830235.656849:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830235.656851:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c64983c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086010368:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830235.783871:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:0.0:1787830235.848776:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830235.848815:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787830235.848843:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787830236.807947:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830236.871969:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787830236.871998:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787830236.872121:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830237.831948:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787830237.895925:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787830237.895962:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830237.895973:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787830238.855938:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787830238.919903:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787830238.919963:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787830238.919977:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830239.879969:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830239.943863:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787830239.943959:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787830239.944018:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787830240.775950:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830240.775963:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787830240.775967:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830240.775979:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3676 00000100:00080000:3.0:1787830240.775990:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830240.776005:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787830240.776055:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498780 x1874672086010496/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830240.776087:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830240.776091:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787830240.776092:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830240.776100:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3676 00000100:00080000:3.0:1787830240.776103:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830240.776107:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830240.776117:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830240.776131:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499680 x1874672086010624/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830240.776142:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787830240.776147:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672086010496:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1787830240.776148:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787830240.776162:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830240.776168:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3676 00000100:00080000:3.0:1787830240.776172:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830240.776176:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830240.776182:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830240.776194:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a940 x1874672086010752/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830240.776205:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830240.776208:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787830240.776212:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1787830240.776213:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1787830240.776221:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3676 00000100:00080000:3.0:1787830240.776225:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830240.776230:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830240.776236:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:3.0:1787830240.776248:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b0c0 x1874672086010880/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830240.776257:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830240.776260:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787830240.776262:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830240.776266:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3676 00000100:00080000:3.0:1787830240.776270:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830240.776276:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787830240.776277:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830240/real 1787830240] req@ffff9e35c6498780 x1874672086010496/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830265 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1787830240.776291:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:3.0:1787830240.776307:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a580 x1874672086011008/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830240.776318:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787830240.776319:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498780 x1874672086010496/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830265 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830240.776325:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787830240.776327:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830240.776341:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787830240.776346:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3676 00000100:00080000:1.0:1787830240.776347:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830240.776354:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672086010496:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1787830240.776380:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086010624:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787830240.776404:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830240.776418:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830240/real 1787830240] req@ffff9e35c6499680 x1874672086010624/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830265 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830240.776429:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499680 x1874672086010624/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830265 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830240.776438:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830240.776441:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830240.776444:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086010624:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787830240.776449:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086010752:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787830240.776468:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830240.776479:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830240/real 1787830240] req@ffff9e35c649a940 x1874672086010752/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830265 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830240.776491:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a940 x1874672086010752/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830265 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830240.776504:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830240.776507:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830240.776510:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086010752:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787830240.776515:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086010880:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787830240.776534:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830240.776547:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830240/real 1787830240] req@ffff9e35c649b0c0 x1874672086010880/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830265 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830240.776559:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1787830240.776560:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b0c0 x1874672086010880/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830265 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830240.776571:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830240.776576:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:3.0:1787830240.776576:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787830240.776584:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086010880:192.168.203.141@tcp:8:kworker.0 00000008:00100000:3.0:1787830240.776586:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830240.776608:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086011008:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787830240.776611:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499e00 x1874672086011136/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830240.776625:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830240.776628:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787830240.776640:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830240.776646:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3676 00000100:00080000:3.0:1787830240.776650:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830240.776653:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830240.776657:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830240.776668:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499a40 x1874672086011264/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830240.776675:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830240.776678:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787830240.776680:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830240.776683:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3676 00000100:00080000:3.0:1787830240.776685:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830240.776692:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:1.0:1787830240.776694:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:3.0:1787830240.776701:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:1.0:1787830240.776714:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830240/real 1787830240] req@ffff9e35c649a580 x1874672086011008/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830265 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787830240.776716:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498b40 x1874672086011392/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787830240.776742:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a580 x1874672086011008/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830265 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830240.776745:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830240.776748:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787830240.776749:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830240.776753:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3676 00000100:00080000:3.0:1787830240.776758:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830240.776766:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:1.0:1787830240.776771:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000008:00100000:3.0:1787830240.776772:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00080000:1.0:1787830240.776779:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830240.776785:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086011008:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787830240.776788:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498f00 x1874672086011520/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787830240.776854:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086011136:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787830240.776880:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830240.776893:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830240/real 1787830240] req@ffff9e35c6499e00 x1874672086011136/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830265 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830240.776906:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499e00 x1874672086011136/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830265 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830240.776913:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830240.776916:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830240.776920:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086011136:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787830240.776925:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086011264:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787830240.776943:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830240.776953:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830240/real 1787830240] req@ffff9e35c6499a40 x1874672086011264/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830265 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830240.776964:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499a40 x1874672086011264/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830265 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830240.776972:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830240.776975:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830240.776978:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086011264:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787830240.776984:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086011392:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787830240.777012:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830240.777024:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830240/real 1787830240] req@ffff9e35c6498b40 x1874672086011392/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830265 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830240.777035:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498b40 x1874672086011392/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830265 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830240.777043:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830240.777046:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830240.777050:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086011392:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830240.777056:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086011520:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787830240.777076:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830240.777086:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830240/real 1787830240] req@ffff9e35c6498f00 x1874672086011520/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830265 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830240.777098:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498f00 x1874672086011520/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830265 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830240.777105:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830240.777107:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830240.777110:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086011520:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830240.903991:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830240.967927:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787830240.968075:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787830240.968086:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830241.927994:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787830241.991931:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787830241.992085:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787830241.992127:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830242.952071:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787830243.016035:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787830243.017388:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787830243.017437:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787830243.975945:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787830244.039988:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787830244.040128:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787830244.040759:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787830245.000943:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830245.063919:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:0.0:1787830245.063959:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787830245.064275:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787830245.896595:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830245.896609:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787830245.896613:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830245.896624:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3681 00000100:00080000:1.0:1787830245.896633:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830245.896641:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787830245.896673:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6940 x1874672086011648/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830245.896705:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830245.896709:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787830245.896711:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830245.896715:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3681 00000100:00080000:1.0:1787830245.896718:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830245.896721:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830245.896730:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830245.896743:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f70c0 x1874672086011776/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830245.896752:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830245.896755:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787830245.896757:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830245.896761:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3681 00000100:00080000:1.0:1787830245.896764:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830245.896768:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830245.896772:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830245.896782:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4b40 x1874672086011904/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830245.896791:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830245.896794:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787830245.896796:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830245.896838:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3681 00000100:00080000:1.0:1787830245.896842:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830245.896846:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830245.896850:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:1.0:1787830245.896858:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5e00 x1874672086012032/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830245.896866:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830245.896869:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787830245.896871:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830245.896874:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3681 00000100:00080000:1.0:1787830245.896876:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830245.896878:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830245.896881:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:1.0:1787830245.896888:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5a40 x1874672086012160/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830245.896893:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830245.896896:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787830245.896897:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830245.896900:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3681 00000100:00080000:1.0:1787830245.896903:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830245.896906:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830245.896910:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830245.896916:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4f00 x1874672086012288/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830245.896921:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830245.896923:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787830245.896924:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830245.896926:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3681 00000100:00080000:1.0:1787830245.896928:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830245.896931:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830245.896934:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830245.896942:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4000 x1874672086012416/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830245.896948:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830245.896950:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787830245.896950:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830245.896952:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3681 00000100:00080000:1.0:1787830245.896954:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830245.896956:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830245.896959:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:1.0:1787830245.896966:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f43c0 x1874672086012544/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830245.896978:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830245.896979:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787830245.896980:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830245.896983:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3681 00000100:00080000:1.0:1787830245.896985:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830245.896987:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830245.896990:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:1.0:1787830245.896999:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f61c0 x1874672086012672/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830245.897843:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672086011648:192.168.203.141@tcp:250:kworker.0 00000800:00000100:0.0:1787830245.897919:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830245.897971:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830245/real 1787830245] req@ffff9e35da6f6940 x1874672086011648/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830270 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830245.897988:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6940 x1874672086011648/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830270 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830245.898140:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830245.898147:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830245.898151:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672086011648:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1787830245.898160:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086011776:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830245.898189:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830245.898213:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830245/real 1787830245] req@ffff9e35da6f70c0 x1874672086011776/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830270 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830245.898227:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f70c0 x1874672086011776/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830270 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830245.898236:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830245.898239:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830245.898242:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086011776:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830245.898248:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086011904:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830245.898267:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830245.898277:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830245/real 1787830245] req@ffff9e35da6f4b40 x1874672086011904/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830270 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830245.898288:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4b40 x1874672086011904/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830270 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830245.898296:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830245.898300:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830245.898303:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086011904:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830245.898308:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086012032:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830245.898328:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830245.898338:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830245/real 1787830245] req@ffff9e35da6f5e00 x1874672086012032/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830270 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830245.898356:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5e00 x1874672086012032/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830270 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830245.898364:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830245.898367:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830245.898370:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086012032:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830245.898376:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086012160:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830245.898393:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830245.898402:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830245/real 1787830245] req@ffff9e35da6f5a40 x1874672086012160/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830270 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830245.898413:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5a40 x1874672086012160/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830270 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830245.898422:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830245.898424:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830245.898427:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086012160:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830245.898432:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086012288:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830245.898448:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830245.898457:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830245/real 1787830245] req@ffff9e35da6f4f00 x1874672086012288/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830270 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830245.898468:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4f00 x1874672086012288/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830270 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830245.898482:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830245.898484:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830245.898487:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086012288:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830245.898492:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086012416:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830245.898507:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830245.898516:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830245/real 1787830245] req@ffff9e35da6f4000 x1874672086012416/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830270 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830245.898526:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4000 x1874672086012416/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830270 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830245.898533:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830245.898535:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830245.898538:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086012416:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830245.898543:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086012544:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830245.898570:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830245.898580:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830245/real 1787830245] req@ffff9e35da6f43c0 x1874672086012544/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830270 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830245.898591:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f43c0 x1874672086012544/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830270 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830245.898599:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830245.898602:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830245.898606:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086012544:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830245.898612:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086012672:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830245.898629:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830245.898640:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830245/real 1787830245] req@ffff9e35da6f61c0 x1874672086012672/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830270 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830245.898650:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f61c0 x1874672086012672/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830270 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830245.898657:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830245.898661:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830245.898664:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086012672:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830246.024573:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787830246.087918:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787830246.087989:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787830246.088522:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830247.048183:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830247.112478:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:0.0:1787830247.112530:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787830247.112617:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787830248.071908:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830248.135926:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787830248.135926:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787830248.135994:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830249.095919:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830249.159967:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787830249.160170:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830249.160380:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787830250.119930:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830250.183922:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787830250.183922:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787830250.184085:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787830251.015921:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830251.015932:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787830251.015936:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830251.015947:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3686 00000100:00080000:1.0:1787830251.015957:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830251.015975:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787830251.016010:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7840 x1874672086012800/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830251.016046:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830251.016050:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787830251.016052:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830251.016056:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3686 00000100:00080000:1.0:1787830251.016061:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830251.016064:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830251.016075:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830251.016089:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6d00 x1874672086012928/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830251.016096:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830251.016098:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787830251.016100:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830251.016103:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3686 00000100:00080000:1.0:1787830251.016106:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830251.016108:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830251.016113:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830251.016125:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4780 x1874672086013056/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830251.016134:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830251.016138:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787830251.016140:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830251.016144:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3686 00000100:00080000:1.0:1787830251.016154:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830251.016159:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830251.016164:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:3.0:1787830251.016173:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672086012800:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1787830251.016178:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6580 x1874672086013184/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830251.016189:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830251.016192:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787830251.016193:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830251.016197:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3686 00000100:00080000:1.0:1787830251.016200:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830251.016204:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830251.016209:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:1.0:1787830251.016220:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7480 x1874672086013312/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830251.016229:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830251.016232:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787830251.016234:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830251.016238:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3686 00000100:00080000:1.0:1787830251.016243:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830251.016252:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830251.016257:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830251.016269:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f61c0 x1874672086013440/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830251.016284:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1787830251.016287:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086012928:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787830251.016289:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787830251.016293:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830251.016311:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3686 00000100:00080000:1.0:1787830251.016316:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830251.016323:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787830251.016324:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086013056:192.168.203.141@tcp:38:kworker.0 00000008:00100000:1.0:1787830251.016329:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830251.016344:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f43c0 x1874672086013568/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830251.016353:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830251.016356:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787830251.016358:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1787830251.016363:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086013184:192.168.203.141@tcp:8:kworker.0 00000100:00080000:1.0:1787830251.016364:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3686 00000100:00080000:1.0:1787830251.016370:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830251.016375:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830251.016380:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:1.0:1787830251.016392:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4000 x1874672086013696/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787830251.016397:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086013312:192.168.203.141@tcp:8:kworker.0 00000100:00080000:1.0:1787830251.016408:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830251.016411:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787830251.016414:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830251.016419:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3686 00000100:00100000:3.0:1787830251.016428:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086013440:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787830251.016429:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830251.016437:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830251.016446:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:3.0:1787830251.016457:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086013568:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787830251.016460:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4f00 x1874672086013824/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787830251.016489:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086013696:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787830251.016512:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086013824:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830251.143928:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830251.207946:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787830251.208286:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787830251.208293:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787830252.168073:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787830252.232076:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787830252.232111:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830252.232163:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787830253.191947:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787830253.255937:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787830253.256295:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787830253.256316:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000400:00000100:0.0:1787830254.091782:0:2399:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.141:988 00000400:00000100:0.0:1787830254.091882:0:2399:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.141@tcp at host 192.168.203.141 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1787830254.091898:0:2399:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:0.0:1787830254.091952:0:2399:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000800:00000100:0.0:1787830254.091963:0:2399:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:0.0:1787830254.091971:0:2399:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:0.0:1787830254.091979:0:2399:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:0.0:1787830254.091985:0:2399:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:0.0:1787830254.091992:0:2399:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:0.0:1787830254.091999:0:2399:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:0.0:1787830254.092006:0:2399:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:1.0:1787830254.092193:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830251/real 1787830254] req@ffff9e35da6f4000 x1874672086013696/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830276 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830254.092219:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4000 x1874672086013696/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830276 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830254.092232:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830254.092237:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830254.092242:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086013696:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830254.092250:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830251/real 1787830254] req@ffff9e35da6f4f00 x1874672086013824/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830276 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830254.092263:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4f00 x1874672086013824/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830276 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830254.092272:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830254.092275:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830254.092278:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086013824:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830254.092284:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830251/real 1787830254] req@ffff9e35da6f6580 x1874672086013184/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830276 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830254.092295:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6580 x1874672086013184/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830276 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830254.092303:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830254.092306:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830254.092310:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086013184:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830254.092316:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830251/real 1787830254] req@ffff9e35da6f7480 x1874672086013312/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830276 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830254.092327:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7480 x1874672086013312/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830276 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830254.092335:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830254.092338:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830254.092341:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086013312:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830254.092347:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830251/real 1787830254] req@ffff9e35da6f61c0 x1874672086013440/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830276 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830254.092358:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f61c0 x1874672086013440/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830276 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830254.092366:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830254.092370:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830254.092373:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086013440:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787830254.093517:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830251/real 1787830254] req@ffff9e35da6f43c0 x1874672086013568/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830276 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830254.093541:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f43c0 x1874672086013568/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830276 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830254.094564:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830254.094573:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830254.094578:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086013568:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787830254.094590:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830251/real 1787830254] req@ffff9e35da6f7840 x1874672086012800/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830276 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830254.094610:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7840 x1874672086012800/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830276 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830254.094619:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830254.094622:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830254.094625:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672086012800:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1787830254.094632:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830251/real 1787830254] req@ffff9e35da6f6d00 x1874672086012928/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830276 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830254.094643:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6d00 x1874672086012928/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830276 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830254.094650:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830254.094653:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830254.094656:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086012928:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787830254.094662:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830251/real 1787830254] req@ffff9e35da6f4780 x1874672086013056/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830276 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830254.094673:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4780 x1874672086013056/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830276 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830254.094679:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830254.094682:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830254.094685:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086013056:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787830254.216009:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787830254.279914:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787830254.280227:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787830254.281554:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830255.241077:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830255.304072:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787830255.304079:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830255.304089:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00080000:2.0:1787830256.136069:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830256.136082:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787830256.136085:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830256.136096:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3691 00000100:00080000:2.0:1787830256.136106:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830256.136114:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787830256.136146:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498ed00 x1874672086013952/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787830256.136211:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672086013952:192.168.203.141@tcp:250:kworker.0 00000800:00000100:1.0:1787830256.136906:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1787830256.136974:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830256.136979:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787830256.136981:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830256.136986:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3691 00000100:00080000:2.0:1787830256.136991:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830256.136995:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830256.137007:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830256.137026:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c000 x1874672086014080/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787830256.137403:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830256/real 1787830256] req@ffff9e35c498ed00 x1874672086013952/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830281 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830256.137423:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498ed00 x1874672086013952/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830281 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830256.137433:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830256.137438:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830256.137442:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672086013952:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1787830256.137471:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086014080:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787830256.137512:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830256.137525:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830256/real 1787830256] req@ffff9e35c498c000 x1874672086014080/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830281 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830256.137535:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c000 x1874672086014080/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830281 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830256.137543:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830256.137547:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830256.137550:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086014080:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1787830256.137673:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830256.137677:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787830256.137679:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830256.137683:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3691 00000100:00080000:2.0:1787830256.137687:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830256.137691:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830256.137697:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830256.137719:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498de00 x1874672086014208/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830256.137730:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830256.137732:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787830256.137734:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830256.137737:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3691 00000100:00080000:2.0:1787830256.137740:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830256.137744:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830256.137748:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:2.0:1787830256.137765:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cb40 x1874672086014336/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830256.137773:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830256.137776:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787830256.137778:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830256.137781:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3691 00000100:00080000:2.0:1787830256.137784:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830256.137787:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830256.137791:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:2.0:1787830256.137821:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498d680 x1874672086014464/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830256.137830:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830256.137833:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787830256.137834:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830256.137838:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3691 00000100:00080000:2.0:1787830256.137840:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830256.137851:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830256.137855:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830256.137865:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e940 x1874672086014592/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830256.137878:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830256.137880:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787830256.137882:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830256.137885:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3691 00000100:00080000:2.0:1787830256.137888:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830256.137891:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830256.137894:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830256.137902:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c000 x1874672086014720/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830256.137908:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830256.137911:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787830256.137913:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830256.137915:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3691 00000100:00080000:2.0:1787830256.137918:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830256.137921:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830256.137925:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:2.0:1787830256.137933:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498ed00 x1874672086014848/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830256.137940:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830256.137942:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787830256.137944:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830256.137947:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3691 00000100:00080000:2.0:1787830256.137950:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830256.137952:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830256.137956:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:2.0:1787830256.137964:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498fc00 x1874672086014976/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787830256.138360:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086014208:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787830256.138397:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830256.138414:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830256/real 1787830256] req@ffff9e35c498de00 x1874672086014208/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830281 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830256.138426:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498de00 x1874672086014208/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830281 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830256.138434:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830256.138437:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830256.138440:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086014208:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787830256.138446:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086014336:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787830256.138465:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830256.138476:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830256/real 1787830256] req@ffff9e35c498cb40 x1874672086014336/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830281 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830256.138500:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498cb40 x1874672086014336/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830281 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830256.138510:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830256.138513:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830256.138516:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086014336:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830256.138521:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086014464:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787830256.138540:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830256.138551:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830256/real 1787830256] req@ffff9e35c498d680 x1874672086014464/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830281 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830256.138562:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498d680 x1874672086014464/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830281 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830256.138570:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830256.138573:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830256.138576:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086014464:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830256.138583:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086014592:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787830256.138600:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830256.138610:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830256/real 1787830256] req@ffff9e35c498e940 x1874672086014592/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830281 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830256.138621:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e940 x1874672086014592/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830281 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830256.138629:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830256.138632:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830256.138635:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086014592:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787830256.138640:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086014720:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787830256.138658:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830256.138668:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830256/real 1787830256] req@ffff9e35c498c000 x1874672086014720/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830281 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830256.138679:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c000 x1874672086014720/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830281 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830256.138687:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830256.138690:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830256.138693:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086014720:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787830256.138698:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086014848:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787830256.138717:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830256.138728:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830256/real 1787830256] req@ffff9e35c498ed00 x1874672086014848/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830281 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830256.138740:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498ed00 x1874672086014848/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830281 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830256.138749:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830256.138751:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830256.138762:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086014848:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830256.138768:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086014976:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787830256.138785:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830256.138795:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830256/real 1787830256] req@ffff9e35c498fc00 x1874672086014976/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830281 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830256.138838:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498fc00 x1874672086014976/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830281 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830256.138846:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830256.138849:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830256.138852:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086014976:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830256.264378:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830256.327964:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:0.0:1787830256.328064:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830256.328071:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787830257.287945:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830257.351937:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787830257.352021:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787830257.352125:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830258.312328:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787830258.375987:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830258.376507:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787830258.376518:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787830259.335933:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830259.399954:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787830259.399954:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787830259.400021:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830260.359923:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830260.423931:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787830260.423931:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787830260.426678:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787830261.256079:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830261.256090:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787830261.256094:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830261.256105:0:27606:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3696 00000100:00080000:3.0:1787830261.256116:0:27606:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830261.256124:0:27606:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787830261.256163:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498f00 x1874672086015104/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830261.256203:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830261.256207:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787830261.256208:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830261.256218:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3696 00000100:00080000:3.0:1787830261.256222:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830261.256225:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830261.256235:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830261.256247:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498b40 x1874672086015232/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830261.256257:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830261.256260:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787830261.256262:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830261.256265:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3696 00000100:00080000:3.0:1787830261.256268:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830261.256271:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830261.256276:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830261.256299:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499a40 x1874672086015360/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830261.256308:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830261.256311:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787830261.256313:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830261.256316:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3696 00000100:00080000:3.0:1787830261.256319:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830261.256323:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830261.256327:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:3.0:1787830261.256336:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499e00 x1874672086015488/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830261.256343:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830261.256346:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787830261.256348:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830261.256351:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3696 00000100:00080000:3.0:1787830261.256354:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830261.256357:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830261.256362:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:3.0:1787830261.256371:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a580 x1874672086015616/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830261.256378:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830261.256381:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787830261.256383:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830261.256386:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3696 00000100:00080000:3.0:1787830261.256388:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830261.256391:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830261.256395:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830261.256403:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b0c0 x1874672086015744/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830261.256410:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830261.256413:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787830261.256415:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830261.256418:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3696 00000100:00080000:3.0:1787830261.256420:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830261.256424:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830261.256427:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830261.256435:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a940 x1874672086015872/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830261.256442:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830261.256445:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787830261.256460:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830261.256464:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3696 00000100:00080000:3.0:1787830261.256468:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830261.256471:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830261.256475:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:3.0:1787830261.256485:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499680 x1874672086016000/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830261.256494:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830261.256497:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787830261.256498:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787830261.256503:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672086015104:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1787830261.256505:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3696 00000100:00080000:3.0:1787830261.256511:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830261.256514:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830261.256518:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:3.0:1787830261.256529:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498780 x1874672086016128/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787830261.256558:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830261.256589:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830261/real 1787830261] req@ffff9e35c6498f00 x1874672086015104/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830286 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830261.256601:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498f00 x1874672086015104/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830286 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830261.256608:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830261.256611:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830261.256613:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672086015104:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1787830261.256618:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086015232:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787830261.256631:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830261.256638:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830261/real 1787830261] req@ffff9e35c6498b40 x1874672086015232/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830286 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830261.256645:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498b40 x1874672086015232/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830286 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830261.256650:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830261.256652:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830261.256654:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086015232:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787830261.256657:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086015360:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787830261.256666:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830261.256671:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830261/real 1787830261] req@ffff9e35c6499a40 x1874672086015360/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830286 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830261.256680:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499a40 x1874672086015360/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830286 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830261.256687:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830261.256690:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830261.256693:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086015360:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787830261.256698:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086015488:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787830261.256716:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830261.256725:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830261/real 1787830261] req@ffff9e35c6499e00 x1874672086015488/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830286 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830261.256743:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499e00 x1874672086015488/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830286 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830261.256751:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830261.256754:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830261.256756:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086015488:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830261.256760:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086015616:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787830261.256773:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830261.256780:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830261/real 1787830261] req@ffff9e35c649a580 x1874672086015616/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830286 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830261.256787:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a580 x1874672086015616/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830286 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830261.256794:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830261.256823:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830261.256826:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086015616:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830261.256832:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086015744:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787830261.256851:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830261.256861:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830261/real 1787830261] req@ffff9e35c649b0c0 x1874672086015744/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830286 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830261.256871:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b0c0 x1874672086015744/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830286 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830261.256881:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830261.256883:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830261.256885:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086015744:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787830261.256889:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086015872:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787830261.256901:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830261.256907:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830261/real 1787830261] req@ffff9e35c649a940 x1874672086015872/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830286 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830261.256913:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a940 x1874672086015872/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830286 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830261.256918:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830261.256920:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830261.256921:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086015872:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787830261.256945:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086016000:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787830261.256954:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830261.256960:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830261/real 1787830261] req@ffff9e35c6499680 x1874672086016000/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830286 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830261.256966:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499680 x1874672086016000/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830286 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830261.256971:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830261.256972:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830261.256974:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086016000:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830261.256977:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086016128:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787830261.256985:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830261.256990:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830261/real 1787830261] req@ffff9e35c6498780 x1874672086016128/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830286 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830261.256996:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498780 x1874672086016128/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830286 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830261.257001:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830261.257002:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830261.257004:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086016128:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830261.383993:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830261.447934:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787830261.447934:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787830261.448106:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830262.408064:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830262.472011:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787830262.472072:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787830262.472245:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830263.431883:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830263.495927:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787830263.496402:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787830263.496473:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787830264.455969:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830264.519945:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787830264.520087:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787830264.520356:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787830265.480185:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787830265.543921:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787830265.544111:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787830265.544123:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787830266.376264:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830266.376277:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787830266.376281:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830266.376293:0:27606:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3701 00000100:00080000:1.0:1787830266.376305:0:27606:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830266.376314:0:27606:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787830266.376365:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4780 x1874672086016256/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830266.376402:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830266.376406:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787830266.376408:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830266.376424:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3701 00000100:00080000:1.0:1787830266.376427:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830266.376431:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830266.376441:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830266.376454:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6d00 x1874672086016384/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830266.376463:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830266.376466:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787830266.376468:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830266.376471:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3701 00000100:00080000:1.0:1787830266.376474:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830266.376477:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830266.376490:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830266.376502:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7840 x1874672086016512/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830266.376519:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830266.376523:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787830266.376524:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1787830266.376525:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672086016256:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787830266.376527:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3701 00000100:00080000:1.0:1787830266.376531:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830266.376534:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830266.376539:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:1.0:1787830266.376552:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f43c0 x1874672086016640/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830266.376560:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830266.376562:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787830266.376564:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830266.376567:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3701 00000100:00080000:1.0:1787830266.376573:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830266.376576:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830266.376580:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000800:00000100:3.0:1787830266.376582:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830266.376588:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f61c0 x1874672086016768/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830266.376595:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830266.376598:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787830266.376599:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830266.376602:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3701 00000100:00080000:1.0:1787830266.376605:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830266.376608:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830266.376612:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830266.376619:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7480 x1874672086016896/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787830266.376622:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830266/real 1787830266] req@ffff9e35da6f4780 x1874672086016256/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830291 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830266.376627:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830266.376630:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787830266.376632:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830266.376635:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3701 00000100:00080000:1.0:1787830266.376638:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1787830266.376638:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4780 x1874672086016256/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830291 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830266.376641:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830266.376645:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00080000:3.0:1787830266.376649:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1787830266.376652:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6580 x1874672086017024/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830266.376652:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787830266.376656:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672086016256:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787830266.376660:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1787830266.376662:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086016384:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787830266.376663:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787830266.376665:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830266.376669:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3701 00000100:00080000:1.0:1787830266.376671:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830266.376675:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830266.376679:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000800:00000100:3.0:1787830266.376680:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830266.376687:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4f00 x1874672086017152/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787830266.376691:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830266/real 1787830266] req@ffff9e35da6f6d00 x1874672086016384/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830291 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830266.376696:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830266.376699:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787830266.376701:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1787830266.376702:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6d00 x1874672086016384/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830291 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830266.376704:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3701 00000100:00080000:1.0:1787830266.376707:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830266.376710:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:3.0:1787830266.376710:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787830266.376713:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000008:00100000:1.0:1787830266.376714:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:3.0:1787830266.376716:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086016384:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787830266.376721:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4000 x1874672086017280/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787830266.376743:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086016512:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787830266.376760:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787830266.376771:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830266/real 1787830266] req@ffff9e35da6f7840 x1874672086016512/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830291 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787830266.376781:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7840 x1874672086016512/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830291 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830266.376788:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787830266.376791:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787830266.376794:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086016512:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787830266.376832:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086016640:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787830266.376852:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787830266.376863:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830266/real 1787830266] req@ffff9e35da6f43c0 x1874672086016640/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830291 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787830266.376873:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f43c0 x1874672086016640/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830291 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830266.376881:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787830266.376884:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787830266.376887:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086016640:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787830266.376892:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086016768:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787830266.376910:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787830266.376920:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830266/real 1787830266] req@ffff9e35da6f61c0 x1874672086016768/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830291 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787830266.376930:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f61c0 x1874672086016768/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830291 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830266.376937:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787830266.376940:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787830266.376943:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086016768:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787830266.376948:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086016896:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787830266.376966:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787830266.376975:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830266/real 1787830266] req@ffff9e35da6f7480 x1874672086016896/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830291 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787830266.376987:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7480 x1874672086016896/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830291 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830266.376996:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787830266.376999:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787830266.377002:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086016896:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787830266.377007:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086017024:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787830266.377027:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787830266.377037:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830266/real 1787830266] req@ffff9e35da6f6580 x1874672086017024/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830291 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787830266.377047:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6580 x1874672086017024/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830291 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830266.377055:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787830266.377058:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787830266.377060:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086017024:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787830266.377065:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086017152:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787830266.377083:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787830266.377094:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830266/real 1787830266] req@ffff9e35da6f4f00 x1874672086017152/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830291 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787830266.377104:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4f00 x1874672086017152/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830291 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830266.377116:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787830266.377119:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787830266.377122:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086017152:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787830266.377127:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086017280:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787830266.377145:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787830266.377155:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830266/real 1787830266] req@ffff9e35da6f4000 x1874672086017280/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830291 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787830266.377166:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4000 x1874672086017280/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830291 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830266.377173:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787830266.377176:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787830266.377179:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086017280:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830266.504407:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830266.567946:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787830266.567946:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:0.0:1787830266.568154:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830267.528458:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830267.591914:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787830267.591969:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787830267.592498:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787830268.551976:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787830268.615943:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:0.0:1787830268.616134:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787830268.616149:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787830269.575930:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:0.0:1787830269.640012:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830269.640401:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787830269.640568:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787830270.600657:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830270.663922:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787830270.663923:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787830270.664004:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787830271.496340:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830271.496350:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787830271.496362:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830271.496373:0:27606:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3706 00000100:00080000:3.0:1787830271.496392:0:27606:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830271.496400:0:27606:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787830271.496437:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c64983c0 x1874672086017408/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787830271.496501:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c64983c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672086017408:192.168.203.141@tcp:250:kworker.0 00000800:00000100:1.0:1787830271.496557:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1787830271.496585:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830271.496588:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787830271.496590:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830271.496595:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3706 00000100:00080000:3.0:1787830271.496598:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830271.496602:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830271.496612:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830271.496625:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498000 x1874672086017536/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787830271.496818:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830271/real 1787830271] req@ffff9e35c64983c0 x1874672086017408/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830296 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830271.496834:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c64983c0 x1874672086017408/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830296 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830271.496848:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830271.496852:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830271.496856:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c64983c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672086017408:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1787830271.496891:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086017536:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787830271.496915:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830271.496927:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830271/real 1787830271] req@ffff9e35c6498000 x1874672086017536/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830296 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830271.496937:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498000 x1874672086017536/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830296 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830271.496944:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830271.496947:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830271.496949:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086017536:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1787830271.497119:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830271.497123:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787830271.497126:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830271.497131:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3706 00000100:00080000:3.0:1787830271.497137:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830271.497141:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830271.497146:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830271.497182:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a1c0 x1874672086017664/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830271.497203:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830271.497206:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787830271.497208:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830271.497212:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3706 00000100:00080000:3.0:1787830271.497216:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830271.497219:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830271.497224:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:3.0:1787830271.497234:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b840 x1874672086017792/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830271.497242:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830271.497245:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787830271.497247:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787830271.497251:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086017664:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1787830271.497255:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3706 00000100:00080000:3.0:1787830271.497261:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830271.497264:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830271.497269:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:3.0:1787830271.497280:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649bc00 x1874672086017920/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830271.497288:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830271.497291:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787830271.497293:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1787830271.497295:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1787830271.497299:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3706 00000100:00080000:3.0:1787830271.497304:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830271.497307:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830271.497312:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830271.497314:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830271/real 1787830271] req@ffff9e35c649a1c0 x1874672086017664/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830296 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787830271.497323:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b480 x1874672086018048/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830271.497333:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787830271.497334:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a1c0 x1874672086017664/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830296 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830271.497338:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787830271.497341:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830271.497349:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3706 00000100:00080000:1.0:1787830271.497354:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830271.497358:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830271.497362:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086017664:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787830271.497368:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086017792:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787830271.497397:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830271.497407:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830271/real 1787830271] req@ffff9e35c649b840 x1874672086017792/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830296 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830271.497420:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b840 x1874672086017792/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830296 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830271.497422:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830271.497428:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830271.497434:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00080000:1.0:1787830271.497448:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1787830271.497449:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c64992c0 x1874672086018176/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830271.497455:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830271.497462:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086017792:192.168.203.141@tcp:8:kworker.0 00000100:00080000:3.0:1787830271.497466:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830271.497471:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787830271.497473:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830271.497478:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3706 00000100:00080000:3.0:1787830271.497482:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1787830271.497485:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086017920:192.168.203.141@tcp:8:kworker.0 00000100:00080000:3.0:1787830271.497489:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830271.497496:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:3.0:1787830271.497509:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649ad00 x1874672086018304/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787830271.497511:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1787830271.497523:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787830271.497527:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830271/real 1787830271] req@ffff9e35c649bc00 x1874672086017920/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830296 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830271.497528:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787830271.497532:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830271.497541:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3706 00000100:00100000:1.0:1787830271.497547:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649bc00 x1874672086017920/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830296 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830271.497547:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830271.497556:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:1.0:1787830271.497560:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000008:00100000:3.0:1787830271.497562:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00080000:1.0:1787830271.497566:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830271.497572:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086017920:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787830271.497579:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b840 x1874672086018432/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787830271.497581:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086018048:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787830271.497609:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830271.497622:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830271/real 1787830271] req@ffff9e35c649b480 x1874672086018048/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830296 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830271.497634:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b480 x1874672086018048/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830296 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830271.497642:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830271.497645:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830271.497658:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086018048:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787830271.497664:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c64992c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086018176:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787830271.497680:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830271.497688:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830271/real 1787830271] req@ffff9e35c64992c0 x1874672086018176/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830296 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830271.497706:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c64992c0 x1874672086018176/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830296 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830271.497713:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830271.497716:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830271.497719:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c64992c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086018176:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787830271.497733:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086018304:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787830271.497747:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830271.497765:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830271/real 1787830271] req@ffff9e35c649ad00 x1874672086018304/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830296 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830271.497777:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649ad00 x1874672086018304/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830296 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830271.497785:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830271.497788:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830271.497791:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086018304:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830271.497814:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086018432:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787830271.497831:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830271.497840:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830271/real 1787830271] req@ffff9e35c649b840 x1874672086018432/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830296 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830271.497851:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b840 x1874672086018432/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830296 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830271.497875:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830271.497878:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830271.497881:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086018432:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830271.624715:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830271.687886:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787830271.687951:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:0.0:1787830271.688026:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830272.647951:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830272.711913:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787830272.711955:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787830272.711990:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830273.671940:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830273.735923:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787830273.736275:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787830273.737315:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830274.695932:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830274.759935:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787830274.759942:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787830274.760195:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830275.720135:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830275.784255:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787830275.784326:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830275.784487:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00080000:3.0:1787830276.615961:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830276.615989:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787830276.615993:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830276.616003:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3711 00000100:00080000:3.0:1787830276.616013:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830276.616020:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787830276.616054:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a1c0 x1874672086018560/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830276.616096:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830276.616099:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787830276.616109:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830276.616114:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3711 00000100:00080000:3.0:1787830276.616118:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830276.616122:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830276.616132:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830276.616146:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498000 x1874672086018688/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787830276.616151:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672086018560:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1787830276.616163:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830276.616167:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787830276.616169:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830276.616173:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3711 00000100:00080000:3.0:1787830276.616177:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830276.616181:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830276.616186:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830276.616197:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c64983c0 x1874672086018816/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830276.616206:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830276.616209:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787830276.616211:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830276.616215:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3711 00000100:00080000:3.0:1787830276.616219:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830276.616224:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:1.0:1787830276.616226:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:3.0:1787830276.616229:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:3.0:1787830276.616241:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498780 x1874672086018944/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830276.616250:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830276.616253:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787830276.616255:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830276.616259:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3711 00000100:00080000:3.0:1787830276.616263:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830276.616266:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830276.616272:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:3.0:1787830276.616285:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499680 x1874672086019072/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787830276.616290:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830276/real 1787830276] req@ffff9e35c649a1c0 x1874672086018560/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830301 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830276.616299:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830276.616306:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787830276.616308:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830276.616314:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3711 00000100:00100000:1.0:1787830276.616315:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a1c0 x1874672086018560/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830301 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830276.616319:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830276.616327:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:1.0:1787830276.616332:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:3.0:1787830276.616334:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00080000:1.0:1787830276.616354:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787830276.616359:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a940 x1874672086019200/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787830276.616361:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672086018560:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1787830276.616376:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830276.616380:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787830276.616382:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830276.616387:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3711 00000100:00080000:3.0:1787830276.616391:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1787830276.616393:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086018688:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1787830276.616397:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830276.616404:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830276.616417:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b0c0 x1874672086019328/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787830276.616425:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1787830276.616431:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830276.616435:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787830276.616438:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787830276.616444:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830276/real 1787830276] req@ffff9e35c6498000 x1874672086018688/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830301 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830276.616444:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3711 00000100:00080000:3.0:1787830276.616449:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830276.616458:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830276.616464:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:1.0:1787830276.616466:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498000 x1874672086018688/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830301 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787830276.616476:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a580 x1874672086019456/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830276.616478:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830276.616485:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830276.616488:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086018688:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787830276.616494:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c64983c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086018816:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1787830276.616506:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830276.616510:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787830276.616513:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1787830276.616518:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1787830276.616522:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3711 00000100:00080000:3.0:1787830276.616527:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1787830276.616533:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830276/real 1787830276] req@ffff9e35c64983c0 x1874672086018816/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830301 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830276.616533:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830276.616541:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:1.0:1787830276.616554:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c64983c0 x1874672086018816/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830301 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787830276.616559:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499e00 x1874672086019584/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830276.616573:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830276.616579:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830276.616584:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c64983c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086018816:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787830276.616591:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086018944:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787830276.616612:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830276.616624:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830276/real 1787830276] req@ffff9e35c6498780 x1874672086018944/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830301 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830276.616635:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498780 x1874672086018944/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830301 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830276.616643:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830276.616646:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830276.616650:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086018944:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830276.616655:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086019072:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787830276.616671:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830276.616681:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830276/real 1787830276] req@ffff9e35c6499680 x1874672086019072/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830301 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830276.616691:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499680 x1874672086019072/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830301 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830276.616699:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830276.616702:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830276.616705:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086019072:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830276.616710:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086019200:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787830276.616726:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830276.616735:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830276/real 1787830276] req@ffff9e35c649a940 x1874672086019200/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830301 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830276.616760:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a940 x1874672086019200/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830301 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830276.616769:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830276.616772:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830276.616775:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086019200:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787830276.616830:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086019328:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787830276.616863:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830276.616875:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830276/real 1787830276] req@ffff9e35c649b0c0 x1874672086019328/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830301 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830276.616886:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b0c0 x1874672086019328/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830301 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830276.616894:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830276.616897:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830276.616899:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086019328:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787830276.616905:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086019456:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787830276.616924:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830276.616934:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830276/real 1787830276] req@ffff9e35c649a580 x1874672086019456/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830301 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830276.616944:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a580 x1874672086019456/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830301 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830276.616952:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830276.616955:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830276.616958:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086019456:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830276.616964:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086019584:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787830276.616979:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830276.616988:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830276/real 1787830276] req@ffff9e35c6499e00 x1874672086019584/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830301 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830276.616999:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499e00 x1874672086019584/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830301 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830276.617006:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830276.617008:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830276.617011:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086019584:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830276.743939:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830276.807954:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787830276.808022:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787830276.808068:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830277.768006:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830277.831956:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787830277.831973:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:0.0:1787830277.832201:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830278.794059:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830278.855931:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787830278.855950:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787830278.855980:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830279.815941:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787830279.879933:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787830279.879980:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787830279.879981:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830280.842394:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830280.903982:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787830280.904038:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:0.0:1787830280.904453:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787830281.736016:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830281.736027:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787830281.736031:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830281.736042:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3717 00000100:00080000:2.0:1787830281.736052:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830281.736060:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787830281.736108:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e580 x1874672086019712/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830281.736138:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830281.736142:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787830281.736144:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830281.736147:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3717 00000100:00080000:2.0:1787830281.736150:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830281.736154:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830281.736164:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830281.736175:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e1c0 x1874672086019840/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830281.736184:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830281.736187:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787830281.736189:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830281.736192:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3717 00000100:00080000:2.0:1787830281.736195:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830281.736198:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830281.736202:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830281.736221:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cf00 x1874672086019968/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830281.736235:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830281.736238:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787830281.736239:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830281.736243:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3717 00000100:00080000:2.0:1787830281.736246:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830281.736249:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830281.736253:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:2.0:1787830281.736264:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f0c0 x1874672086020096/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830281.736283:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787830281.736286:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672086019712:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1787830281.736287:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787830281.736291:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830281.736299:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3717 00000100:00080000:2.0:1787830281.736302:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830281.736317:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830281.736322:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:2.0:1787830281.736333:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f840 x1874672086020224/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830281.736342:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830281.736346:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787830281.736347:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830281.736351:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3717 00000100:00080000:2.0:1787830281.736356:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830281.736359:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830281.736363:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1787830281.736369:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787830281.736376:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c3c0 x1874672086020352/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830281.736386:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830281.736389:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787830281.736391:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830281.736394:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3717 00000100:00080000:2.0:1787830281.736398:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830281.736402:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830281.736407:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830281.736415:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830281/real 1787830281] req@ffff9e35c498e580 x1874672086019712/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830306 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830281.736419:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c780 x1874672086020480/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830281.736442:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830281.736445:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787830281.736449:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787830281.736450:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e580 x1874672086019712/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830306 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830281.736458:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3717 00000100:00080000:2.0:1787830281.736465:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830281.736470:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830281.736471:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:1.0:1787830281.736480:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000008:00100000:2.0:1787830281.736480:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:1.0:1787830281.736487:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672086019712:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1787830281.736494:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f480 x1874672086020608/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787830281.736497:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086019840:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1787830281.736516:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830281.736519:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000800:00000100:1.0:1787830281.736525:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1787830281.736533:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830281.736539:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3717 00000100:00100000:1.0:1787830281.736540:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830281/real 1787830281] req@ffff9e35c498e1c0 x1874672086019840/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830306 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830281.736545:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830281.736552:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830281.736559:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:1.0:1787830281.736561:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e1c0 x1874672086019840/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830306 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830281.736572:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:2.0:1787830281.736573:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498fc00 x1874672086020736/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830281.736579:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830281.736587:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086019840:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787830281.736596:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086019968:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787830281.736618:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830281.736630:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830281/real 1787830281] req@ffff9e35c498cf00 x1874672086019968/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830306 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830281.736642:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498cf00 x1874672086019968/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830306 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830281.736651:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830281.736654:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830281.736657:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086019968:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787830281.736685:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086020096:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787830281.736707:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830281.736717:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830281/real 1787830281] req@ffff9e35c498f0c0 x1874672086020096/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830306 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830281.736729:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f0c0 x1874672086020096/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830306 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830281.736737:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830281.736740:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830281.736743:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086020096:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830281.736749:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086020224:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787830281.736767:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830281.736777:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830281/real 1787830281] req@ffff9e35c498f840 x1874672086020224/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830306 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830281.736789:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f840 x1874672086020224/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830306 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830281.736828:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830281.736832:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830281.736836:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086020224:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830281.736859:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086020352:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787830281.736879:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830281.736890:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830281/real 1787830281] req@ffff9e35c498c3c0 x1874672086020352/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830306 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830281.736909:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c3c0 x1874672086020352/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830306 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830281.736917:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830281.736920:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830281.736924:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086020352:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787830281.736929:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086020480:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787830281.736947:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830281.736957:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830281/real 1787830281] req@ffff9e35c498c780 x1874672086020480/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830306 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830281.736976:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c780 x1874672086020480/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830306 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830281.736984:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830281.736987:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830281.736990:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086020480:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787830281.736996:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086020608:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787830281.737013:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830281.737023:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830281/real 1787830281] req@ffff9e35c498f480 x1874672086020608/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830306 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830281.737032:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f480 x1874672086020608/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830306 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830281.737058:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830281.737061:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830281.737064:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086020608:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830281.737070:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086020736:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787830281.737097:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830281.737113:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830281/real 1787830281] req@ffff9e35c498fc00 x1874672086020736/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830306 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830281.737123:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498fc00 x1874672086020736/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830306 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830281.737130:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830281.737133:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830281.737136:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086020736:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830281.864016:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830281.927967:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787830281.928840:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830281.928968:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787830282.887993:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830282.951968:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787830282.951981:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787830282.952017:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830283.911942:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787830283.975901:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787830283.975907:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787830283.976028:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830284.935997:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787830284.999929:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787830284.999930:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787830285.000195:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830285.959953:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787830286.023943:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787830286.024097:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787830286.024762:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00080000:3.0:1787830286.856116:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830286.856124:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787830286.856129:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830286.856140:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3722 00000100:00080000:3.0:1787830286.856151:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830286.856160:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787830286.856194:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499a40 x1874672086020864/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830286.856219:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830286.856223:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787830286.856225:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830286.856229:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3722 00000100:00080000:3.0:1787830286.856233:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830286.856237:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830286.856248:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830286.856258:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672086020864:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1787830286.856264:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498b40 x1874672086020992/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830286.856289:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830286.856292:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787830286.856294:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830286.856299:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3722 00000100:00080000:3.0:1787830286.856303:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830286.856306:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830286.856312:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830286.856323:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498f00 x1874672086021120/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830286.856332:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1787830286.856336:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1787830286.856337:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787830286.856341:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830286.856346:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3722 00000100:00080000:3.0:1787830286.856350:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830286.856354:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830286.856359:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:3.0:1787830286.856373:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499e00 x1874672086021248/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830286.856381:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830286.856385:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787830286.856386:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830286.856390:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3722 00000100:00080000:3.0:1787830286.856393:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1787830286.856397:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830286/real 1787830286] req@ffff9e35c6499a40 x1874672086020864/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830311 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830286.856401:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830286.856415:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:1.0:1787830286.856422:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499a40 x1874672086020864/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830311 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787830286.856432:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a580 x1874672086021376/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830286.856436:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830286.856445:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:3.0:1787830286.856447:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787830286.856452:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672086020864:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1787830286.856453:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787830286.856457:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830286.856464:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3722 00000100:00080000:3.0:1787830286.856468:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830286.856472:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830286.856476:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830286.856486:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086020992:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787830286.856490:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b0c0 x1874672086021504/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830286.856502:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830286.856505:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787830286.856507:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830286.856510:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3722 00000100:00080000:3.0:1787830286.856514:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1787830286.856518:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1787830286.856521:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830286.856528:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830286.856537:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830286/real 1787830286] req@ffff9e35c6498b40 x1874672086020992/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830311 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787830286.856542:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a940 x1874672086021632/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830286.856562:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830286.856566:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:1.0:1787830286.856567:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498b40 x1874672086020992/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830311 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830286.856568:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830286.856576:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3722 00000100:00080000:3.0:1787830286.856580:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830286.856582:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830286.856586:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00080000:1.0:1787830286.856597:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1787830286.856599:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499680 x1874672086021760/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830286.856605:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830286.856611:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086020992:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787830286.856618:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086021120:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1787830286.856631:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830286.856634:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787830286.856636:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830286.856641:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3722 00000100:00080000:3.0:1787830286.856646:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830286.856650:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:1.0:1787830286.856655:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:3.0:1787830286.856657:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:1.0:1787830286.856670:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830286/real 1787830286] req@ffff9e35c6498f00 x1874672086021120/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830311 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787830286.856673:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498780 x1874672086021888/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787830286.856708:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498f00 x1874672086021120/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830311 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830286.856716:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830286.856719:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830286.856721:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086021120:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787830286.856727:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086021248:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787830286.856748:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830286.856759:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830286/real 1787830286] req@ffff9e35c6499e00 x1874672086021248/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830311 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830286.856771:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499e00 x1874672086021248/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830311 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830286.856778:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830286.856792:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830286.856826:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086021248:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830286.856835:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086021376:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787830286.856856:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830286.856870:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830286/real 1787830286] req@ffff9e35c649a580 x1874672086021376/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830311 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830286.856882:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a580 x1874672086021376/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830311 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830286.856894:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830286.856910:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830286.856914:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086021376:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830286.856934:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086021504:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787830286.856953:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830286.856964:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830286/real 1787830286] req@ffff9e35c649b0c0 x1874672086021504/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830311 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830286.856974:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b0c0 x1874672086021504/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830311 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830286.856982:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830286.856984:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830286.856987:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086021504:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787830286.856992:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086021632:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787830286.857008:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830286.857018:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830286/real 1787830286] req@ffff9e35c649a940 x1874672086021632/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830311 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830286.857029:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a940 x1874672086021632/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830311 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830286.857037:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830286.857041:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830286.857045:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086021632:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787830286.857052:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086021760:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787830286.857072:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830286.857081:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830286/real 1787830286] req@ffff9e35c6499680 x1874672086021760/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830311 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830286.857090:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499680 x1874672086021760/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830311 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830286.857097:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830286.857099:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830286.857102:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086021760:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830286.857107:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086021888:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787830286.857122:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830286.857131:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830286/real 1787830286] req@ffff9e35c6498780 x1874672086021888/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830311 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830286.857142:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498780 x1874672086021888/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830311 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830286.857155:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830286.857158:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830286.857161:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086021888:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830286.983965:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830287.047942:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787830287.047942:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787830287.048015:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830288.007999:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830288.072010:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:0.0:1787830288.072086:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830288.072996:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787830289.032381:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830289.095897:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787830289.095916:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787830289.096128:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787830290.056242:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787830290.119944:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787830290.119961:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787830290.120279:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830291.079940:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830291.143916:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787830291.143960:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787830291.144392:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00080000:0.0:1787830291.975934:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830291.975943:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787830291.975946:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830291.975954:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3727 00000100:00080000:0.0:1787830291.975969:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830291.975976:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787830291.976010:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7192c0 x1874672086022016/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830291.976034:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830291.976037:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787830291.976039:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830291.976044:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3727 00000100:00080000:0.0:1787830291.976047:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830291.976051:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830291.976060:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830291.976075:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718f00 x1874672086022144/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830291.976095:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830291.976098:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787830291.976100:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830291.976104:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3727 00000100:00080000:0.0:1787830291.976107:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830291.976111:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830291.976115:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830291.976125:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a940 x1874672086022272/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787830291.976126:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672086022016:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1787830291.976134:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830291.976137:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787830291.976138:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830291.976142:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3727 00000100:00080000:0.0:1787830291.976144:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830291.976147:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830291.976151:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:0.0:1787830291.976159:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718780 x1874672086022400/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830291.976166:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830291.976169:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787830291.976170:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830291.976173:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3727 00000100:00080000:0.0:1787830291.976177:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1787830291.976179:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787830291.976180:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830291.976183:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:0.0:1787830291.976192:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da719a40 x1874672086022528/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830291.976199:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830291.976202:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787830291.976204:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830291.976207:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3727 00000100:00080000:0.0:1787830291.976209:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830291.976213:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830291.976216:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830291.976220:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830291/real 1787830291] req@ffff9e35da7192c0 x1874672086022016/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830316 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830291.976224:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7183c0 x1874672086022656/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830291.976232:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830291.976248:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787830291.976250:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787830291.976250:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7192c0 x1874672086022016/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830316 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830291.976253:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3727 00000100:00080000:0.0:1787830291.976257:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830291.976260:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:1.0:1787830291.976260:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830291.976263:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000008:00100000:0.0:1787830291.976264:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830291.976266:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672086022016:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1787830291.976273:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718000 x1874672086022784/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787830291.976273:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086022144:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787830291.976286:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830291.976289:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787830291.976291:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1787830291.976292:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787830291.976294:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3727 00000100:00080000:0.0:1787830291.976297:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830291.976301:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787830291.976304:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830291/real 1787830291] req@ffff9e35da718f00 x1874672086022144/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830316 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1787830291.976305:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:0.0:1787830291.976314:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a580 x1874672086022912/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787830291.976315:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672086022144/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830316 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830291.976322:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830291.976323:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830291.976325:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787830291.976327:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830291.976327:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830291.976329:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086022144:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787830291.976330:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3727 00000100:00080000:0.0:1787830291.976333:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830291.976336:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830291.976340:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:0.0:1787830291.976348:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718b40 x1874672086023040/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787830291.976365:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086022272:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787830291.976383:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830291.976395:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830291/real 1787830291] req@ffff9e35da71a940 x1874672086022272/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830316 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830291.976406:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672086022272/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830316 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830291.976414:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830291.976417:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830291.976420:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086022272:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787830291.976425:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086022400:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787830291.976445:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830291.976456:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830291/real 1787830291] req@ffff9e35da718780 x1874672086022400/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830316 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830291.976467:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718780 x1874672086022400/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830316 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830291.976475:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830291.976478:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830291.976481:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086022400:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830291.976487:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086022528:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787830291.976504:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830291.976515:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830291/real 1787830291] req@ffff9e35da719a40 x1874672086022528/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830316 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830291.976526:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672086022528/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830316 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830291.976534:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830291.976537:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830291.976540:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086022528:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830291.976546:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086022656:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787830291.976563:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830291.976573:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830291/real 1787830291] req@ffff9e35da7183c0 x1874672086022656/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830316 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830291.976584:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672086022656/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830316 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830291.976593:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830291.976595:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830291.976598:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086022656:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787830291.976603:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086022784:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787830291.976620:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830291.976630:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830291/real 1787830291] req@ffff9e35da718000 x1874672086022784/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830316 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830291.976647:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718000 x1874672086022784/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830316 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830291.976654:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830291.976657:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830291.976660:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086022784:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787830291.976665:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086022912:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787830291.976697:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830291.976707:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830291/real 1787830291] req@ffff9e35da71a580 x1874672086022912/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830316 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830291.976717:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672086022912/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830316 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830291.976725:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830291.976729:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830291.976731:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086022912:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830291.976737:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086023040:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787830291.976753:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830291.976764:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830291/real 1787830291] req@ffff9e35da718b40 x1874672086023040/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830316 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830291.976776:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718b40 x1874672086023040/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830316 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830291.976788:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830291.976791:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830291.976793:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086023040:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830292.104003:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787830292.167946:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:3.0:1787830292.168124:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787830292.168231:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830293.127949:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787830293.191916:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:3.0:1787830293.191917:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787830293.192018:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830294.152131:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830294.216093:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787830294.216098:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:0.0:1787830294.216122:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830295.175999:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830295.239915:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787830295.239915:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787830295.239962:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830296.199940:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830296.263925:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787830296.263934:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:0.0:1787830296.265286:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787830297.096000:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830297.096011:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787830297.096015:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830297.096026:0:27606:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3732 00000100:00080000:3.0:1787830297.096037:0:27606:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830297.096044:0:27606:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787830297.096082:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c64983c0 x1874672086023168/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830297.096117:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830297.096120:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787830297.096122:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830297.096126:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3732 00000100:00080000:3.0:1787830297.096130:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830297.096133:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830297.096143:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830297.096156:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498000 x1874672086023296/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830297.096164:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830297.096167:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787830297.096168:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830297.096172:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3732 00000100:00080000:3.0:1787830297.096174:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830297.096179:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830297.096183:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830297.096192:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a1c0 x1874672086023424/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830297.096218:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830297.096222:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787830297.096224:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830297.096227:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3732 00000100:00080000:3.0:1787830297.096230:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830297.096233:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830297.096238:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:3.0:1787830297.096315:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b840 x1874672086023552/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830297.096325:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830297.096328:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787830297.096329:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830297.096333:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3732 00000100:00080000:3.0:1787830297.096337:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830297.096340:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830297.096344:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:3.0:1787830297.096356:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649ad00 x1874672086023680/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830297.096364:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830297.096367:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787830297.096369:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830297.096373:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3732 00000100:00080000:3.0:1787830297.096375:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830297.096378:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830297.096382:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830297.096391:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c64992c0 x1874672086023808/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830297.096399:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830297.096402:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787830297.096404:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830297.096407:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3732 00000100:00080000:3.0:1787830297.096410:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830297.096413:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830297.096417:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830297.096425:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b480 x1874672086023936/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830297.096434:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830297.096437:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787830297.096438:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830297.096441:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3732 00000100:00080000:3.0:1787830297.096444:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830297.096447:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830297.096451:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:3.0:1787830297.096458:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649bc00 x1874672086024064/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830297.096464:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830297.096467:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787830297.096469:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830297.096471:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3732 00000100:00080000:3.0:1787830297.096473:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830297.096476:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830297.096479:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:3.0:1787830297.096488:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498780 x1874672086024192/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787830297.096579:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c64983c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672086023168:192.168.203.141@tcp:250:kworker.0 00000800:00000100:1.0:1787830297.096633:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830297.096670:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830297/real 1787830297] req@ffff9e35c64983c0 x1874672086023168/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830322 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830297.096685:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c64983c0 x1874672086023168/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830322 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830297.096693:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830297.096696:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830297.096699:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c64983c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672086023168:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1787830297.096705:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086023296:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787830297.096721:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830297.096731:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830297/real 1787830297] req@ffff9e35c6498000 x1874672086023296/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830322 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830297.096740:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498000 x1874672086023296/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830322 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830297.096747:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830297.096749:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830297.096752:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086023296:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787830297.096757:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086023424:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787830297.096770:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830297.096779:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830297/real 1787830297] req@ffff9e35c649a1c0 x1874672086023424/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830322 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830297.096789:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a1c0 x1874672086023424/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830322 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830297.096827:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830297.096831:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830297.096834:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086023424:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787830297.096839:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086023552:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787830297.096856:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830297.096866:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830297/real 1787830297] req@ffff9e35c649b840 x1874672086023552/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830322 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830297.096876:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b840 x1874672086023552/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830322 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830297.096887:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830297.096890:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830297.096892:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086023552:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830297.096897:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086023680:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787830297.096914:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830297.096923:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830297/real 1787830297] req@ffff9e35c649ad00 x1874672086023680/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830322 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830297.096934:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649ad00 x1874672086023680/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830322 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830297.096941:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830297.096944:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830297.096946:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086023680:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830297.096951:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c64992c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086023808:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787830297.096969:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830297.096978:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830297/real 1787830297] req@ffff9e35c64992c0 x1874672086023808/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830322 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830297.096989:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c64992c0 x1874672086023808/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830322 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830297.096997:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830297.097000:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830297.097003:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c64992c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086023808:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787830297.097008:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086023936:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787830297.097026:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830297.097035:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830297/real 1787830297] req@ffff9e35c649b480 x1874672086023936/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830322 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830297.097044:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b480 x1874672086023936/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830322 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830297.097051:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830297.097054:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830297.097056:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086023936:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787830297.097061:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086024064:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787830297.097076:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830297.097085:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830297/real 1787830297] req@ffff9e35c649bc00 x1874672086024064/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830322 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830297.097094:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649bc00 x1874672086024064/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830322 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830297.097101:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830297.097104:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830297.097106:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086024064:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830297.097112:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086024192:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787830297.097128:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830297.097139:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830297/real 1787830297] req@ffff9e35c6498780 x1874672086024192/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830322 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830297.097148:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498780 x1874672086024192/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830322 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830297.097155:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830297.097158:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830297.097160:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086024192:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830297.223977:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:0.0:1787830297.288132:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787830297.288141:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787830297.288375:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787830298.247938:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787830298.311928:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787830298.312072:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787830298.312110:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830299.273287:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830299.335939:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787830299.336020:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787830299.336028:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830300.295977:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830300.359927:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787830300.359973:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787830300.360011:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830301.319946:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787830301.384362:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787830301.385636:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787830301.385636:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00080000:1.0:1787830302.216239:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830302.216251:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787830302.216255:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830302.216266:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3737 00000100:00080000:1.0:1787830302.216276:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830302.216284:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787830302.216318:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5a40 x1874672086024320/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830302.216357:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830302.216361:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787830302.216363:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830302.216368:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3737 00000100:00080000:1.0:1787830302.216371:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830302.216374:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830302.216383:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830302.216393:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5e00 x1874672086024448/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830302.216401:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830302.216404:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787830302.216405:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830302.216408:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3737 00000100:00080000:1.0:1787830302.216411:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830302.216414:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830302.216417:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830302.216426:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4b40 x1874672086024576/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830302.216433:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830302.216436:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787830302.216438:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830302.216441:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3737 00000100:00080000:1.0:1787830302.216444:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830302.216447:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830302.216451:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:1.0:1787830302.216459:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f70c0 x1874672086024704/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830302.216467:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830302.216470:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787830302.216472:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830302.216475:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3737 00000100:00080000:1.0:1787830302.216478:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830302.216481:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830302.216485:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:1.0:1787830302.216495:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6940 x1874672086024832/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830302.216505:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830302.216508:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787830302.216512:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1787830302.216512:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672086024320:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787830302.216521:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3737 00000100:00080000:1.0:1787830302.216525:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830302.216529:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830302.216534:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830302.216549:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4000 x1874672086024960/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830302.216563:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830302.216566:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787830302.216568:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:3.0:1787830302.216573:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787830302.216574:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3737 00000100:00080000:1.0:1787830302.216580:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830302.216584:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830302.216589:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830302.216600:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4f00 x1874672086025088/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830302.216608:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830302.216612:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787830302.216613:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830302.216618:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3737 00000100:00100000:3.0:1787830302.216621:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830302/real 1787830302] req@ffff9e35da6f5a40 x1874672086024320/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830327 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830302.216623:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830302.216631:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830302.216638:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:3.0:1787830302.216645:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5a40 x1874672086024320/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830327 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830302.216650:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6580 x1874672086025216/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830302.216663:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830302.216665:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830302.216670:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:1.0:1787830302.216671:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787830302.216675:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1787830302.216678:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672086024320:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787830302.216683:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3737 00000100:00080000:1.0:1787830302.216688:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1787830302.216690:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086024448:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787830302.216695:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830302.216701:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:1.0:1787830302.216714:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7480 x1874672086025344/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787830302.216722:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787830302.216739:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830302/real 1787830302] req@ffff9e35da6f5e00 x1874672086024448/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830327 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787830302.216751:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5e00 x1874672086024448/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830327 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830302.216761:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787830302.216764:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787830302.216767:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086024448:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787830302.216773:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086024576:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787830302.216790:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787830302.216961:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830302/real 1787830302] req@ffff9e35da6f4b40 x1874672086024576/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830327 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787830302.216983:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4b40 x1874672086024576/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830327 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830302.216990:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787830302.216993:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787830302.216996:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086024576:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787830302.217002:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086024704:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787830302.217024:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787830302.217034:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830302/real 1787830302] req@ffff9e35da6f70c0 x1874672086024704/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830327 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787830302.217045:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f70c0 x1874672086024704/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830327 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830302.217052:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787830302.217054:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787830302.217057:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086024704:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787830302.217099:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086024832:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787830302.217119:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787830302.217129:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830302/real 1787830302] req@ffff9e35da6f6940 x1874672086024832/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830327 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787830302.217139:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6940 x1874672086024832/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830327 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830302.217152:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787830302.217155:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787830302.217158:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086024832:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787830302.217175:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086024960:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787830302.217195:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787830302.217206:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830302/real 1787830302] req@ffff9e35da6f4000 x1874672086024960/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830327 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787830302.217217:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4000 x1874672086024960/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830327 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830302.217224:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787830302.217227:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787830302.217231:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086024960:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787830302.217236:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086025088:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787830302.217254:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787830302.217265:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830302/real 1787830302] req@ffff9e35da6f4f00 x1874672086025088/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830327 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787830302.217276:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4f00 x1874672086025088/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830327 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830302.217282:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787830302.217285:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787830302.217288:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086025088:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787830302.217293:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086025216:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787830302.217312:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787830302.217322:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830302/real 1787830302] req@ffff9e35da6f6580 x1874672086025216/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830327 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787830302.217332:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6580 x1874672086025216/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830327 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830302.217339:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787830302.217342:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787830302.217345:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086025216:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787830302.217350:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086025344:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787830302.217366:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787830302.217376:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830302/real 1787830302] req@ffff9e35da6f7480 x1874672086025344/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830327 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787830302.217496:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7480 x1874672086025344/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830327 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830302.217503:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787830302.217506:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787830302.217509:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086025344:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830302.344079:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787830302.407943:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787830302.408140:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787830302.408962:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830303.368898:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787830303.431952:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787830303.432002:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830303.432205:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787830304.391951:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787830304.455936:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787830304.455983:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830304.456514:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787830305.415983:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787830305.479949:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787830305.481510:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787830305.481552:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830306.440038:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830306.503924:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787830306.504041:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787830306.504057:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787830307.336182:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830307.336195:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787830307.336199:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830307.336269:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3742 00000100:00080000:0.0:1787830307.336280:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830307.336287:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787830307.336325:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a1c0 x1874672086025472/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787830307.336403:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672086025472:192.168.203.141@tcp:250:kworker.0 00000800:00000100:3.0:1787830307.336462:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787830307.336491:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830307.336495:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787830307.336497:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830307.336501:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3742 00000100:00080000:0.0:1787830307.336505:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830307.336509:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830307.336519:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830307.336535:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718b40 x1874672086025600/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787830307.336603:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830307/real 1787830307] req@ffff9e35da71a1c0 x1874672086025472/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830332 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787830307.336620:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a1c0 x1874672086025472/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830332 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830307.336630:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787830307.336633:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787830307.336637:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672086025472:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1787830307.336658:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086025600:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787830307.336679:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787830307.336691:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830307/real 1787830307] req@ffff9e35da718b40 x1874672086025600/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830332 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787830307.336702:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718b40 x1874672086025600/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830332 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830307.336710:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787830307.336713:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787830307.336716:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086025600:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787830307.336941:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830307.336946:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787830307.336947:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830307.336953:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3742 00000100:00080000:0.0:1787830307.336957:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830307.336961:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830307.336967:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830307.336981:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a580 x1874672086025728/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787830307.337024:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086025728:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787830307.337048:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787830307.337071:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830307.337074:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787830307.337077:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830307.337081:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3742 00000100:00080000:0.0:1787830307.337085:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830307.337089:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830307.337095:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:0.0:1787830307.337107:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718000 x1874672086025856/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787830307.337152:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830307/real 1787830307] req@ffff9e35da71a580 x1874672086025728/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830332 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787830307.337171:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672086025728/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830332 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830307.337180:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787830307.337183:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787830307.337186:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086025728:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787830307.337199:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086025856:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787830307.337274:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787830307.337286:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830307/real 1787830307] req@ffff9e35da718000 x1874672086025856/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830332 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787830307.337299:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718000 x1874672086025856/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830332 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830307.337307:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787830307.337311:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787830307.337314:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086025856:192.168.203.141@tcp:8:kworker.0 00000100:00080000:0.0:1787830307.337399:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830307.337403:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787830307.337405:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830307.337410:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3742 00000100:00080000:0.0:1787830307.337415:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830307.337419:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830307.337424:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:0.0:1787830307.337437:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7183c0 x1874672086025984/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787830307.337479:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086025984:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787830307.337503:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787830307.337525:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830307.337529:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787830307.337531:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830307.337535:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3742 00000100:00080000:0.0:1787830307.337539:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830307.337543:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830307.337548:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830307.337560:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da719a40 x1874672086026112/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787830307.337655:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830307/real 1787830307] req@ffff9e35da7183c0 x1874672086025984/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830332 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787830307.337668:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672086025984/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830332 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830307.337683:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787830307.337687:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787830307.337690:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086025984:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787830307.337702:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086026112:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787830307.337723:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787830307.337735:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830307/real 1787830307] req@ffff9e35da719a40 x1874672086026112/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830332 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787830307.337746:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672086026112/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830332 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830307.337754:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787830307.337757:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787830307.337760:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086026112:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787830307.337922:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830307.337926:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787830307.337927:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830307.337932:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3742 00000100:00080000:0.0:1787830307.337936:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830307.337940:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830307.337946:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830307.337958:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718780 x1874672086026240/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787830307.338003:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086026240:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787830307.338026:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787830307.338046:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830307.338051:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787830307.338052:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830307.338057:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3742 00000100:00080000:0.0:1787830307.338061:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830307.338065:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830307.338070:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:0.0:1787830307.338083:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a940 x1874672086026368/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787830307.338122:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830307/real 1787830307] req@ffff9e35da718780 x1874672086026240/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830332 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787830307.338146:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718780 x1874672086026240/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830332 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830307.338154:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787830307.338157:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787830307.338161:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086026240:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787830307.338173:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086026368:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787830307.338194:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787830307.338261:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830307.338265:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787830307.338267:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830307.338271:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3742 00000100:00080000:0.0:1787830307.338276:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830307.338280:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830307.338285:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:0.0:1787830307.338298:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718f00 x1874672086026496/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787830307.338350:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830307/real 1787830307] req@ffff9e35da71a940 x1874672086026368/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830332 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787830307.338364:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672086026368/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830332 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830307.338373:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787830307.338376:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787830307.338380:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086026368:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787830307.338392:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086026496:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787830307.338414:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787830307.338425:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830307/real 1787830307] req@ffff9e35da718f00 x1874672086026496/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830332 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787830307.338436:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672086026496/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830332 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830307.338444:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787830307.338447:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787830307.338450:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086026496:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830307.464463:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830307.528064:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787830307.528332:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:0.0:1787830307.528408:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830308.487949:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830308.551974:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787830308.551979:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787830308.555178:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830309.511881:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830309.575926:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787830309.576020:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787830309.576276:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830310.535966:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830310.600115:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787830310.600140:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:0.0:1787830310.601149:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830311.560073:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830311.623936:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787830311.623990:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787830311.624083:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787830312.455962:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830312.455972:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787830312.455976:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830312.455988:0:27606:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3747 00000100:00080000:1.0:1787830312.455999:0:27606:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830312.456007:0:27606:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787830312.456043:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f61c0 x1874672086026624/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830312.456076:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830312.456080:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787830312.456082:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830312.456085:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3747 00000100:00080000:1.0:1787830312.456089:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830312.456106:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830312.456116:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830312.456128:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f43c0 x1874672086026752/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830312.456138:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830312.456141:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787830312.456142:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830312.456146:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3747 00000100:00080000:1.0:1787830312.456149:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830312.456152:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830312.456156:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830312.456165:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7840 x1874672086026880/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830312.456180:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830312.456183:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787830312.456184:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830312.456188:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3747 00000100:00080000:1.0:1787830312.456191:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830312.456195:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830312.456204:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:1.0:1787830312.456214:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6d00 x1874672086027008/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830312.456222:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830312.456225:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787830312.456227:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830312.456230:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3747 00000100:00080000:1.0:1787830312.456234:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830312.456237:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830312.456242:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:1.0:1787830312.456251:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4780 x1874672086027136/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830312.456259:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830312.456262:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787830312.456263:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830312.456267:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3747 00000100:00080000:1.0:1787830312.456270:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830312.456273:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830312.456277:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830312.456287:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7480 x1874672086027264/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830312.456295:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830312.456298:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787830312.456300:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830312.456303:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3747 00000100:00080000:1.0:1787830312.456306:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830312.456309:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830312.456312:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830312.456320:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6580 x1874672086027392/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830312.456328:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830312.456331:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787830312.456332:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830312.456336:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3747 00000100:00080000:1.0:1787830312.456338:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830312.456342:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830312.456345:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:1.0:1787830312.456364:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4f00 x1874672086027520/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830312.456378:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830312.456380:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787830312.456382:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830312.456385:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3747 00000100:00080000:1.0:1787830312.456388:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830312.456391:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830312.456395:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:1.0:1787830312.456405:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4000 x1874672086027648/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787830312.456485:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672086026624:192.168.203.141@tcp:250:kworker.0 00000800:00000100:3.0:1787830312.456556:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787830312.456594:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830312/real 1787830312] req@ffff9e35da6f61c0 x1874672086026624/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830337 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787830312.456608:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f61c0 x1874672086026624/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830337 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830312.456617:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787830312.456621:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787830312.456624:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672086026624:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1787830312.456630:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086026752:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787830312.456648:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787830312.456658:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830312/real 1787830312] req@ffff9e35da6f43c0 x1874672086026752/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830337 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787830312.456669:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f43c0 x1874672086026752/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830337 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830312.456676:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787830312.456679:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787830312.456682:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086026752:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787830312.456687:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086026880:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787830312.456702:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787830312.456710:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830312/real 1787830312] req@ffff9e35da6f7840 x1874672086026880/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830337 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787830312.456737:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7840 x1874672086026880/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830337 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830312.456745:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787830312.456748:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787830312.456751:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086026880:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787830312.456756:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086027008:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787830312.456771:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787830312.456780:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830312/real 1787830312] req@ffff9e35da6f6d00 x1874672086027008/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830337 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787830312.456790:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6d00 x1874672086027008/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830337 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830312.456819:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787830312.456823:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787830312.456826:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086027008:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787830312.456831:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086027136:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787830312.456848:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787830312.456858:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830312/real 1787830312] req@ffff9e35da6f4780 x1874672086027136/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830337 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787830312.456868:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4780 x1874672086027136/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830337 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830312.456881:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787830312.456884:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787830312.456886:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086027136:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787830312.456892:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086027264:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787830312.456908:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787830312.456917:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830312/real 1787830312] req@ffff9e35da6f7480 x1874672086027264/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830337 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787830312.456926:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7480 x1874672086027264/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830337 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830312.456932:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787830312.456935:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787830312.456937:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086027264:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787830312.456942:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086027392:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787830312.456958:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787830312.456966:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830312/real 1787830312] req@ffff9e35da6f6580 x1874672086027392/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830337 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787830312.456977:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6580 x1874672086027392/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830337 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830312.456983:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787830312.456985:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787830312.456988:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086027392:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787830312.456993:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086027520:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787830312.457008:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787830312.457016:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830312/real 1787830312] req@ffff9e35da6f4f00 x1874672086027520/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830337 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787830312.457026:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4f00 x1874672086027520/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830337 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830312.457034:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787830312.457037:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787830312.457040:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086027520:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787830312.457044:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086027648:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787830312.457059:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787830312.457068:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830312/real 1787830312] req@ffff9e35da6f4000 x1874672086027648/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830337 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787830312.457078:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4000 x1874672086027648/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830337 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830312.457085:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787830312.457088:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787830312.457103:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086027648:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830312.583932:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830312.647923:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787830312.647934:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787830312.647965:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830313.607946:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830313.671923:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787830313.671923:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:0.0:1787830313.672272:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830314.632127:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830314.696025:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787830314.696224:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830314.697045:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787830315.655910:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:3.0:1787830315.720415:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787830315.720832:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830315.720902:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787830316.680076:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830316.743902:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787830316.743934:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830316.744334:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00080000:2.0:1787830317.575998:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830317.576009:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787830317.576013:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830317.576023:0:27606:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3752 00000100:00080000:2.0:1787830317.576034:0:27606:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830317.576041:0:27606:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787830317.576088:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498ed00 x1874672086027776/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830317.576122:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830317.576125:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787830317.576127:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830317.576131:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3752 00000100:00080000:2.0:1787830317.576135:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830317.576138:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830317.576148:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830317.576161:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c000 x1874672086027904/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830317.576170:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830317.576173:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787830317.576174:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830317.576178:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3752 00000100:00080000:2.0:1787830317.576181:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830317.576184:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830317.576188:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830317.576197:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e940 x1874672086028032/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830317.576205:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830317.576208:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787830317.576210:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830317.576214:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3752 00000100:00080000:2.0:1787830317.576217:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830317.576220:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830317.576225:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:2.0:1787830317.576236:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498d680 x1874672086028160/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830317.576244:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830317.576247:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787830317.576249:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830317.576253:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3752 00000100:00080000:2.0:1787830317.576256:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830317.576260:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830317.576264:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:2.0:1787830317.576274:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cb40 x1874672086028288/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830317.576282:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830317.576286:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787830317.576287:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830317.576291:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3752 00000100:00080000:2.0:1787830317.576294:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830317.576298:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830317.576302:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830317.576311:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498de00 x1874672086028416/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830317.576319:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830317.576321:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787830317.576323:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830317.576326:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3752 00000100:00080000:2.0:1787830317.576329:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830317.576332:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830317.576335:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830317.576344:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498fc00 x1874672086028544/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830317.576353:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830317.576356:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787830317.576358:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830317.576362:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3752 00000100:00080000:2.0:1787830317.576365:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830317.576369:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830317.576373:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:2.0:1787830317.576382:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f480 x1874672086028672/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830317.576390:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830317.576392:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787830317.576394:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830317.576397:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3752 00000100:00080000:2.0:1787830317.576400:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830317.576404:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830317.576408:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:2.0:1787830317.576416:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c780 x1874672086028800/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787830317.576566:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672086027776:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1787830317.576664:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086027904:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787830317.576694:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086028032:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787830317.576723:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086028160:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787830317.576747:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086028288:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787830317.576768:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086028416:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787830317.576789:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086028544:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787830317.576840:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086028672:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787830317.576860:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086028800:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830317.703943:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830317.767969:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787830317.767976:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787830317.767990:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830318.727945:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830318.791927:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787830318.791928:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787830318.792137:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830319.751956:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830319.815941:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787830319.816054:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787830319.816057:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000400:00000100:1.0:1787830320.648900:0:2396:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.141:988 00000400:00000100:1.0:1787830320.648961:0:2396:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.141@tcp at host 192.168.203.141 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1787830320.648979:0:2396:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:1.0:1787830320.649041:0:2396:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000800:00000100:1.0:1787830320.649054:0:2396:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:1.0:1787830320.649062:0:2396:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:1.0:1787830320.649069:0:2396:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:1.0:1787830320.649076:0:2396:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:1.0:1787830320.649083:0:2396:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:1.0:1787830320.649088:0:2396:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:1.0:1787830320.649095:0:2396:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:3.0:1787830320.649187:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830317/real 1787830320] req@ffff9e35c498ed00 x1874672086027776/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830342 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787830320.649212:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498ed00 x1874672086027776/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830342 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830320.649224:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787830320.649229:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787830320.649232:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672086027776:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1787830320.649241:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830317/real 1787830320] req@ffff9e35c498c000 x1874672086027904/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830342 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787830320.649254:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c000 x1874672086027904/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830342 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830320.649267:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787830320.649271:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787830320.649274:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086027904:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787830320.649281:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830317/real 1787830320] req@ffff9e35c498e940 x1874672086028032/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830342 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787830320.649293:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e940 x1874672086028032/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830342 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830320.649300:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787830320.649303:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787830320.649307:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086028032:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787830320.649312:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830317/real 1787830320] req@ffff9e35c498d680 x1874672086028160/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830342 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787830320.649323:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498d680 x1874672086028160/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830342 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830320.649332:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787830320.649335:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787830320.649338:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086028160:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787830320.649354:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830317/real 1787830320] req@ffff9e35c498cb40 x1874672086028288/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830342 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787830320.649364:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498cb40 x1874672086028288/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830342 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830320.649372:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787830320.649375:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787830320.649378:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086028288:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787830320.649383:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830317/real 1787830320] req@ffff9e35c498de00 x1874672086028416/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830342 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787830320.649394:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498de00 x1874672086028416/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830342 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830320.649400:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787830320.649404:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787830320.649406:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086028416:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787830320.649412:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830317/real 1787830320] req@ffff9e35c498fc00 x1874672086028544/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830342 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787830320.649421:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498fc00 x1874672086028544/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830342 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830320.649427:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787830320.649430:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787830320.649432:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086028544:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787830320.649438:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830317/real 1787830320] req@ffff9e35c498f480 x1874672086028672/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830342 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787830320.649447:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f480 x1874672086028672/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830342 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830320.649454:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787830320.649457:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787830320.649460:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086028672:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787830320.649466:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830317/real 1787830320] req@ffff9e35c498c780 x1874672086028800/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830342 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787830320.649475:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c780 x1874672086028800/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830342 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830320.649482:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787830320.649484:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787830320.649487:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086028800:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830320.776148:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830320.839938:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787830320.840178:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787830320.843618:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787830321.799962:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787830321.865187:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787830321.865431:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787830321.865619:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787830322.697865:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830322.697875:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787830322.697878:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830322.697888:0:27606:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3758 00000100:00080000:2.0:1787830322.697897:0:27606:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830322.697903:0:27606:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787830322.697937:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c3c0 x1874672086028928/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830322.697972:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830322.697976:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787830322.697978:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830322.697981:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3758 00000100:00080000:2.0:1787830322.697985:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830322.697988:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830322.697998:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830322.698010:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f840 x1874672086029056/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830322.698029:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830322.698032:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787830322.698034:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830322.698038:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3758 00000100:00080000:2.0:1787830322.698041:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830322.698044:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830322.698049:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830322.698058:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f0c0 x1874672086029184/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830322.698066:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830322.698069:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787830322.698070:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830322.698073:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3758 00000100:00080000:2.0:1787830322.698075:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830322.698077:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830322.698081:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:2.0:1787830322.698088:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cf00 x1874672086029312/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830322.698096:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830322.698098:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787830322.698100:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830322.698103:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3758 00000100:00080000:2.0:1787830322.698105:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830322.698107:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830322.698111:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:2.0:1787830322.698120:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e1c0 x1874672086029440/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830322.698128:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830322.698131:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787830322.698132:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830322.698135:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3758 00000100:00080000:2.0:1787830322.698138:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830322.698142:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830322.698145:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830322.698152:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e580 x1874672086029568/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830322.698160:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830322.698162:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787830322.698164:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830322.698213:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3758 00000100:00080000:2.0:1787830322.698217:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830322.698220:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830322.698224:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830322.698234:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c780 x1874672086029696/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830322.698243:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830322.698246:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787830322.698247:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830322.698251:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3758 00000100:00080000:2.0:1787830322.698254:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830322.698257:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830322.698261:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:2.0:1787830322.698271:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f480 x1874672086029824/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830322.698284:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830322.698287:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787830322.698573:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830322.698579:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3758 00000100:00080000:2.0:1787830322.698583:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830322.698587:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830322.698592:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:2.0:1787830322.698604:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498fc00 x1874672086029952/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830322.698775:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672086028928:192.168.203.141@tcp:250:kworker.0 00000800:00000100:0.0:1787830322.698891:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830322.698929:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830322/real 1787830322] req@ffff9e35c498c3c0 x1874672086028928/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830347 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830322.698957:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c3c0 x1874672086028928/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830347 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830322.698966:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830322.698970:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830322.698974:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672086028928:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1787830322.698982:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086029056:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830322.699004:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830322.699016:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830322/real 1787830322] req@ffff9e35c498f840 x1874672086029056/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830347 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830322.699037:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f840 x1874672086029056/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830347 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830322.699046:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830322.699049:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830322.699052:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086029056:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830322.699058:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086029184:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830322.699077:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830322.699087:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830322/real 1787830322] req@ffff9e35c498f0c0 x1874672086029184/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830347 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830322.699105:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f0c0 x1874672086029184/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830347 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830322.699113:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830322.699116:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830322.699119:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086029184:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830322.699125:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086029312:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830322.699143:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830322.699153:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830322/real 1787830322] req@ffff9e35c498cf00 x1874672086029312/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830347 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830322.699163:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498cf00 x1874672086029312/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830347 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830322.699171:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830322.699175:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830322.699177:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086029312:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830322.699183:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086029440:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830322.699200:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830322.699210:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830322/real 1787830322] req@ffff9e35c498e1c0 x1874672086029440/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830347 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830322.699219:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e1c0 x1874672086029440/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830347 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830322.699232:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830322.699235:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830322.699238:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086029440:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830322.699243:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086029568:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830322.699259:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830322.699269:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830322/real 1787830322] req@ffff9e35c498e580 x1874672086029568/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830347 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830322.699278:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e580 x1874672086029568/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830347 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830322.699285:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830322.699299:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830322.699303:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086029568:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830322.699308:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086029696:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830322.699325:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830322.699335:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830322/real 1787830322] req@ffff9e35c498c780 x1874672086029696/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830347 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830322.699347:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c780 x1874672086029696/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830347 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830322.699354:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830322.699357:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830322.699361:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086029696:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830322.699366:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086029824:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830322.699383:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830322.699393:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830322/real 1787830322] req@ffff9e35c498f480 x1874672086029824/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830347 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830322.699403:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f480 x1874672086029824/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830347 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830322.699411:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830322.699414:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830322.699417:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086029824:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830322.699423:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086029952:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830322.699439:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830322.699448:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830322/real 1787830322] req@ffff9e35c498fc00 x1874672086029952/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830347 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830322.699458:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498fc00 x1874672086029952/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830347 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830322.699465:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830322.699468:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830322.699471:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086029952:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830322.823942:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830322.887945:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787830322.888065:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787830322.888219:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830323.848589:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787830323.912592:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787830323.912923:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830323.912977:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787830324.871940:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830324.935963:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787830324.937073:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830324.937118:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787830325.896138:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830325.959915:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787830325.959915:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787830325.960065:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830326.919962:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:1.0:1787830326.984105:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787830326.984168:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787830326.984339:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787830327.815897:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830327.815908:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787830327.815912:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830327.815924:0:27606:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3763 00000100:00080000:1.0:1787830327.815933:0:27606:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830327.815941:0:27606:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787830327.815993:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6940 x1874672086030080/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830327.816024:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830327.816028:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787830327.816030:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830327.816033:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3763 00000100:00080000:1.0:1787830327.816036:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830327.816041:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830327.816050:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830327.816063:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f70c0 x1874672086030208/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830327.816084:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830327.816087:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787830327.816089:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830327.816093:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3763 00000100:00080000:1.0:1787830327.816096:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830327.816099:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830327.816126:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830327.816136:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4b40 x1874672086030336/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830327.816149:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830327.816153:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787830327.816154:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830327.816166:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3763 00000100:00080000:1.0:1787830327.816169:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830327.816173:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830327.816177:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:1.0:1787830327.816185:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5e00 x1874672086030464/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830327.816200:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830327.816203:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787830327.816204:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830327.816207:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3763 00000100:00080000:1.0:1787830327.816216:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830327.816219:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830327.816223:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:1.0:1787830327.816249:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5a40 x1874672086030592/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830327.816257:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830327.816279:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787830327.816281:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830327.816285:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3763 00000100:00080000:1.0:1787830327.816288:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830327.816291:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830327.816295:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830327.816305:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4000 x1874672086030720/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830327.816313:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830327.816315:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787830327.816317:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830327.816320:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3763 00000100:00080000:1.0:1787830327.816323:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830327.816326:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830327.816331:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830327.816339:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4f00 x1874672086030848/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830327.816348:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830327.816351:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787830327.816352:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830327.816356:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3763 00000100:00080000:1.0:1787830327.816358:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830327.816362:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830327.816365:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:1.0:1787830327.816374:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6580 x1874672086030976/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830327.816386:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830327.816389:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787830327.816391:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830327.816394:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3763 00000100:00080000:1.0:1787830327.816397:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830327.816405:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830327.816409:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:1.0:1787830327.816418:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7480 x1874672086031104/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830327.816514:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672086030080:192.168.203.141@tcp:250:kworker.0 00000800:00000100:0.0:1787830327.816571:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830327.816608:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830327/real 1787830327] req@ffff9e35da6f6940 x1874672086030080/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830352 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830327.816623:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6940 x1874672086030080/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830352 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830327.816633:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830327.816636:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830327.816640:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672086030080:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1787830327.816646:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086030208:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830327.816667:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830327.816677:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830327/real 1787830327] req@ffff9e35da6f70c0 x1874672086030208/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830352 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830327.816687:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f70c0 x1874672086030208/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830352 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830327.816695:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830327.816698:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830327.816701:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086030208:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830327.816707:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086030336:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830327.816741:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830327.816750:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830327/real 1787830327] req@ffff9e35da6f4b40 x1874672086030336/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830352 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830327.816761:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4b40 x1874672086030336/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830352 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830327.816767:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830327.816770:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830327.816772:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086030336:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830327.816777:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086030464:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830327.816791:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830327.816830:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830327/real 1787830327] req@ffff9e35da6f5e00 x1874672086030464/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830352 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830327.816841:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5e00 x1874672086030464/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830352 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830327.816849:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830327.816851:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830327.816854:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086030464:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830327.816860:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086030592:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830327.816877:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830327.816887:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830327/real 1787830327] req@ffff9e35da6f5a40 x1874672086030592/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830352 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830327.816902:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5a40 x1874672086030592/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830352 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830327.816910:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830327.816913:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830327.816915:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086030592:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830327.816920:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086030720:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830327.816934:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830327.816942:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830327/real 1787830327] req@ffff9e35da6f4000 x1874672086030720/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830352 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830327.816952:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4000 x1874672086030720/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830352 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830327.816958:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830327.816961:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830327.816963:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086030720:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830327.816968:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086030848:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830327.816981:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830327.817001:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830327/real 1787830327] req@ffff9e35da6f4f00 x1874672086030848/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830352 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830327.817011:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4f00 x1874672086030848/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830352 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830327.817022:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830327.817025:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830327.817028:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086030848:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830327.817033:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086030976:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830327.817048:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830327.817056:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830327/real 1787830327] req@ffff9e35da6f6580 x1874672086030976/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830352 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830327.817065:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6580 x1874672086030976/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830352 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830327.817073:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830327.817075:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830327.817078:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086030976:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830327.817084:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086031104:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830327.817097:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830327.817105:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830327/real 1787830327] req@ffff9e35da6f7480 x1874672086031104/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830352 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830327.817115:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7480 x1874672086031104/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830352 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830327.817122:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830327.817125:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830327.817129:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086031104:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830327.944558:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830328.007919:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787830328.007979:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787830328.008022:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830328.967927:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830329.031939:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787830329.032778:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787830329.032895:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830329.991957:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787830330.055990:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830330.056041:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787830330.056578:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787830331.016518:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830331.079914:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787830331.079914:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787830331.080237:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830332.039922:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830332.103915:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787830332.103943:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787830332.103996:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787830332.936994:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830332.937006:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787830332.937011:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830332.937022:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3768 00000100:00080000:2.0:1787830332.937032:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830332.937039:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787830332.937074:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498de00 x1874672086031232/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830332.937107:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830332.937111:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787830332.937113:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830332.937118:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3768 00000100:00080000:2.0:1787830332.937122:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830332.937125:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830332.937135:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830332.937147:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cb40 x1874672086031360/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830332.937156:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830332.937159:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787830332.937160:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830332.937164:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3768 00000100:00080000:2.0:1787830332.937167:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830332.937170:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830332.937174:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830332.937183:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498d680 x1874672086031488/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830332.937192:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830332.937195:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787830332.937197:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830332.937200:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3768 00000100:00080000:2.0:1787830332.937203:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830332.937206:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830332.937210:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:2.0:1787830332.937222:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e940 x1874672086031616/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830332.937230:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830332.937233:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787830332.937234:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830332.937239:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3768 00000100:00080000:2.0:1787830332.937241:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830332.937245:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830332.937249:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:2.0:1787830332.937258:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c000 x1874672086031744/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830332.937267:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830332.937269:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787830332.937271:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830332.937275:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3768 00000100:00080000:2.0:1787830332.937278:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830332.937281:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830332.937285:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830332.937294:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498ed00 x1874672086031872/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830332.937301:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830332.937305:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787830332.937306:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830332.937310:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3768 00000100:00080000:2.0:1787830332.937313:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830332.937317:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830332.937321:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830332.937329:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498fc00 x1874672086032000/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830332.937336:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830332.937339:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787830332.937340:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830332.937344:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3768 00000100:00080000:2.0:1787830332.937346:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830332.937349:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830332.937353:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:2.0:1787830332.937362:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f480 x1874672086032128/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830332.937370:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830332.937372:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787830332.937374:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830332.937377:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3768 00000100:00080000:2.0:1787830332.937380:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830332.937383:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830332.937387:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:2.0:1787830332.937396:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c780 x1874672086032256/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830332.937624:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672086031232:192.168.203.141@tcp:250:kworker.0 00000800:00000100:0.0:1787830332.937853:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830332.937895:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830332/real 1787830332] req@ffff9e35c498de00 x1874672086031232/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830357 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830332.937911:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498de00 x1874672086031232/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830357 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830332.937920:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830332.937923:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830332.937927:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672086031232:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1787830332.937934:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086031360:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830332.937967:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830332.937980:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830332/real 1787830332] req@ffff9e35c498cb40 x1874672086031360/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830357 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830332.937991:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498cb40 x1874672086031360/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830357 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830332.937999:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830332.938001:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830332.938005:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086031360:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830332.938010:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086031488:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830332.938027:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830332.938036:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830332/real 1787830332] req@ffff9e35c498d680 x1874672086031488/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830357 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830332.938045:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498d680 x1874672086031488/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830357 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830332.938052:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830332.938055:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830332.938058:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086031488:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830332.938063:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086031616:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830332.938080:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830332.938090:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830332/real 1787830332] req@ffff9e35c498e940 x1874672086031616/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830357 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830332.938101:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e940 x1874672086031616/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830357 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830332.938109:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830332.938112:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830332.938114:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086031616:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830332.938120:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086031744:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830332.938138:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830332.938148:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830332/real 1787830332] req@ffff9e35c498c000 x1874672086031744/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830357 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830332.938159:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c000 x1874672086031744/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830357 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830332.938166:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830332.938169:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830332.938172:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086031744:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830332.938177:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086031872:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830332.938195:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830332.938205:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830332/real 1787830332] req@ffff9e35c498ed00 x1874672086031872/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830357 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830332.938216:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498ed00 x1874672086031872/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830357 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830332.938224:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830332.938226:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830332.938229:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086031872:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830332.938234:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086032000:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830332.938252:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830332.938261:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830332/real 1787830332] req@ffff9e35c498fc00 x1874672086032000/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830357 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830332.938270:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498fc00 x1874672086032000/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830357 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830332.938278:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830332.938281:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830332.938284:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086032000:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830332.938290:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086032128:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830332.938313:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830332.938322:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830332/real 1787830332] req@ffff9e35c498f480 x1874672086032128/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830357 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830332.938332:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f480 x1874672086032128/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830357 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830332.938339:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830332.938343:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830332.938346:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086032128:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830332.938351:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086032256:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830332.938368:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830332.938377:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830332/real 1787830332] req@ffff9e35c498c780 x1874672086032256/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830357 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830332.938387:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c780 x1874672086032256/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830357 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830332.938394:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830332.938397:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830332.938401:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086032256:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830333.063944:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830333.127937:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787830333.128065:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787830333.128303:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830334.088116:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830334.151919:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787830334.151930:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787830334.152413:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830335.112956:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830335.175942:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787830335.175945:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787830335.176526:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830336.135948:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830336.199937:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787830336.199938:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787830336.199974:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830337.164406:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830337.223948:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787830337.223973:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787830337.223992:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787830338.056526:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830338.056538:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787830338.056543:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830338.056560:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3773 00000100:00080000:2.0:1787830338.056570:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830338.056577:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787830338.056613:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e580 x1874672086032384/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830338.056779:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830338.056784:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787830338.056786:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830338.056790:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3773 00000100:00080000:2.0:1787830338.056795:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830338.056827:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787830338.056836:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672086032384:192.168.203.141@tcp:250:kworker.0 00000008:00100000:2.0:1787830338.056837:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830338.056855:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e1c0 x1874672086032512/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830338.056867:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830338.056871:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787830338.056873:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830338.056877:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3773 00000100:00080000:2.0:1787830338.056881:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830338.056885:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830338.056889:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830338.056909:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cf00 x1874672086032640/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830338.056928:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830338.056932:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787830338.056933:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830338.056937:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3773 00000100:00080000:2.0:1787830338.056940:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830338.056944:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830338.056948:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000800:00000100:0.0:1787830338.056954:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787830338.056963:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f0c0 x1874672086032768/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830338.056972:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830338.056976:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787830338.056981:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830338.056987:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3773 00000100:00080000:2.0:1787830338.056990:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830338.056993:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830338.056998:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:2.0:1787830338.057010:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f840 x1874672086032896/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830338.057013:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830338/real 1787830338] req@ffff9e35c498e580 x1874672086032384/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830363 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830338.057032:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830338.057037:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787830338.057039:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787830338.057043:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e580 x1874672086032384/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830363 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830338.057046:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3773 00000100:00080000:2.0:1787830338.057052:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830338.057059:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:0.0:1787830338.057060:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:2.0:1787830338.057065:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1787830338.057067:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830338.057073:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672086032384:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1787830338.057079:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c3c0 x1874672086033024/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830338.057090:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830338.057092:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787830338.057094:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787830338.057100:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086032512:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1787830338.057100:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3773 00000100:00080000:2.0:1787830338.057105:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830338.057111:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830338.057116:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1787830338.057131:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787830338.057138:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e580 x1874672086033152/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830338.057146:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830338/real 1787830338] req@ffff9e35c498e1c0 x1874672086032512/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830363 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830338.057152:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830338.057159:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787830338.057163:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787830338.057169:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e1c0 x1874672086032512/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830363 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830338.057169:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3773 00000100:00080000:2.0:1787830338.057177:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830338.057184:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:0.0:1787830338.057185:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830338.057190:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830338.057194:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086032512:192.168.203.141@tcp:38:kworker.0 00000008:00100000:2.0:1787830338.057200:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:0.0:1787830338.057205:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086032640:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787830338.057214:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c780 x1874672086033280/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830338.057224:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830338.057228:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787830338.057230:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1787830338.057232:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1787830338.057237:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3773 00000100:00080000:2.0:1787830338.057242:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830338.057246:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787830338.057250:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830338/real 1787830338] req@ffff9e35c498cf00 x1874672086032640/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830363 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1787830338.057253:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:2.0:1787830338.057269:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f480 x1874672086033408/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830338.057272:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498cf00 x1874672086032640/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830363 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830338.057291:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830338.057294:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830338.057297:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086032640:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830338.057303:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086032768:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830338.057324:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830338.057335:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830338/real 1787830338] req@ffff9e35c498f0c0 x1874672086032768/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830363 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830338.057345:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f0c0 x1874672086032768/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830363 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830338.057353:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830338.057356:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830338.057360:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086032768:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830338.057366:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086032896:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830338.057385:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830338.057396:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830338/real 1787830338] req@ffff9e35c498f840 x1874672086032896/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830363 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830338.057408:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f840 x1874672086032896/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830363 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830338.057416:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830338.057419:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830338.057422:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086032896:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830338.057427:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086033024:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830338.057445:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830338.057946:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830338/real 1787830338] req@ffff9e35c498c3c0 x1874672086033024/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830363 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830338.057965:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c3c0 x1874672086033024/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830363 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830338.057974:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830338.057977:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830338.057980:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086033024:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830338.058007:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086033152:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830338.058032:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830338.058045:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830338/real 1787830338] req@ffff9e35c498e580 x1874672086033152/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830363 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830338.058056:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e580 x1874672086033152/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830363 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830338.058064:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830338.058067:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830338.058070:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086033152:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830338.058076:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086033280:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830338.058094:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830338.058104:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830338/real 1787830338] req@ffff9e35c498c780 x1874672086033280/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830363 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830338.058114:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c780 x1874672086033280/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830363 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830338.058121:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830338.058124:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830338.058127:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086033280:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830338.058132:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086033408:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830338.058149:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830338.058159:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830338/real 1787830338] req@ffff9e35c498f480 x1874672086033408/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830363 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830338.058169:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f480 x1874672086033408/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830363 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830338.058176:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830338.058179:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830338.058182:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086033408:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830338.184474:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830338.247939:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787830338.248052:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830338.248362:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787830339.207958:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830339.271963:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787830339.272105:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787830339.272179:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830340.231944:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787830340.295989:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830340.296424:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787830340.296480:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787830341.256227:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830341.320037:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787830341.320410:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787830341.320540:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787830342.280458:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830342.343974:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787830342.343984:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787830342.344100:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00080000:3.0:1787830343.176080:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830343.176092:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787830343.176095:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830343.176106:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3778 00000100:00080000:3.0:1787830343.176115:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830343.176122:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787830343.176217:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499680 x1874672086033536/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830343.176250:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830343.176253:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787830343.176254:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830343.176257:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3778 00000100:00080000:3.0:1787830343.176259:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830343.176262:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830343.176271:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830343.176280:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a940 x1874672086033664/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830343.176290:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830343.176292:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787830343.176293:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830343.176296:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3778 00000100:00080000:3.0:1787830343.176297:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830343.176299:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830343.176302:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830343.176307:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b0c0 x1874672086033792/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830343.176320:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830343.176322:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787830343.176324:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830343.176326:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3778 00000100:00080000:3.0:1787830343.176328:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830343.176330:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830343.176333:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:3.0:1787830343.176339:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a580 x1874672086033920/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830343.176344:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830343.176346:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787830343.176347:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830343.176349:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3778 00000100:00080000:3.0:1787830343.176351:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830343.176353:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830343.176355:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:3.0:1787830343.176360:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499e00 x1874672086034048/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830343.176364:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830343.176366:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787830343.176367:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830343.176369:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3778 00000100:00080000:3.0:1787830343.176371:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830343.176373:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830343.176375:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830343.176379:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498f00 x1874672086034176/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830343.176388:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830343.176390:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787830343.176391:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830343.176393:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3778 00000100:00080000:3.0:1787830343.176394:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830343.176396:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830343.176398:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830343.176402:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498b40 x1874672086034304/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830343.176407:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830343.176408:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787830343.176409:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830343.176411:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3778 00000100:00080000:3.0:1787830343.176412:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830343.176415:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830343.176417:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:3.0:1787830343.176421:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499a40 x1874672086034432/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830343.176426:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830343.176428:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:0.0:1787830343.176559:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672086033536:192.168.203.141@tcp:250:kworker.0 00000800:00000100:0.0:1787830343.176608:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830343.176641:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830343/real 1787830343] req@ffff9e35c6499680 x1874672086033536/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830368 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830343.176652:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499680 x1874672086033536/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830368 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830343.176658:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830343.176660:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830343.176663:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672086033536:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1787830343.176667:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086033664:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830343.176680:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830343.176687:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830343/real 1787830343] req@ffff9e35c649a940 x1874672086033664/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830368 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830343.176693:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a940 x1874672086033664/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830368 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830343.176698:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830343.176701:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830343.176702:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086033664:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830343.176705:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086033792:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830343.176715:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830343.176720:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830343/real 1787830343] req@ffff9e35c649b0c0 x1874672086033792/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830368 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830343.176726:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b0c0 x1874672086033792/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830368 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830343.176731:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830343.176733:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830343.176736:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086033792:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830343.176741:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086033920:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830343.176758:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830343.176768:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830343/real 1787830343] req@ffff9e35c649a580 x1874672086033920/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830368 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830343.176779:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a580 x1874672086033920/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830368 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830343.176787:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830343.176790:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830343.176793:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086033920:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830343.176844:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086034048:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830343.176864:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830343.176872:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830343/real 1787830343] req@ffff9e35c6499e00 x1874672086034048/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830368 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830343.176904:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499e00 x1874672086034048/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830368 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830343.176912:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830343.176915:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830343.176919:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086034048:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830343.176925:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086034176:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830343.176944:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830343.176954:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830343/real 1787830343] req@ffff9e35c6498f00 x1874672086034176/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830368 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830343.176966:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498f00 x1874672086034176/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830368 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830343.176973:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830343.176976:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830343.176979:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086034176:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830343.176984:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086034304:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830343.177002:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830343.177012:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830343/real 1787830343] req@ffff9e35c6498b40 x1874672086034304/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830368 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830343.177022:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498b40 x1874672086034304/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830368 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830343.177033:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830343.177036:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830343.177038:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086034304:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830343.177042:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086034432:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830343.177060:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830343.177071:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830343/real 1787830343] req@ffff9e35c6499a40 x1874672086034432/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830368 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830343.177081:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499a40 x1874672086034432/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830368 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830343.177088:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830343.177091:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830343.177094:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086034432:192.168.203.141@tcp:8:kworker.0 00000100:00080000:3.0:1787830343.177185:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830343.177192:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3778 00000100:00080000:3.0:1787830343.177197:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830343.177201:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830343.177208:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:3.0:1787830343.177225:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499a40 x1874672086034560/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830343.177364:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086034560:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830343.177388:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830343.177398:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830343/real 1787830343] req@ffff9e35c6499a40 x1874672086034560/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830368 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830343.177408:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499a40 x1874672086034560/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830368 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830343.177416:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830343.177419:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830343.177423:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086034560:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830343.303940:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787830343.367950:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787830343.368026:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787830343.368104:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787830344.328113:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830344.391954:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787830344.391954:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787830344.392076:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830345.351943:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787830345.415947:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787830345.416140:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830345.416158:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787830346.376515:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830346.439983:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787830346.439983:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787830346.441060:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830347.399944:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830347.463962:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787830347.463970:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830347.464464:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00080000:2.0:1787830348.296130:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830348.296142:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787830348.296146:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830348.296157:0:27606:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3783 00000100:00080000:2.0:1787830348.296167:0:27606:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830348.296174:0:27606:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787830348.296209:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498fc00 x1874672086034688/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830348.296239:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830348.296243:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787830348.296245:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830348.296250:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3783 00000100:00080000:2.0:1787830348.296253:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830348.296257:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830348.296268:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830348.296285:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498ed00 x1874672086034816/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830348.296294:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830348.296298:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787830348.296299:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830348.296303:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3783 00000100:00080000:2.0:1787830348.296307:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830348.296311:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830348.296316:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830348.296326:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c000 x1874672086034944/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830348.296336:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830348.296339:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787830348.296340:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830348.296344:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3783 00000100:00080000:2.0:1787830348.296612:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830348.296619:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830348.296624:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:2.0:1787830348.296638:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e940 x1874672086035072/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830348.296647:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830348.296650:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787830348.296652:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830348.296656:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3783 00000100:00080000:2.0:1787830348.296660:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830348.296663:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830348.296667:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:2.0:1787830348.296680:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498d680 x1874672086035200/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830348.296687:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830348.296690:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787830348.296692:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830348.296696:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3783 00000100:00080000:2.0:1787830348.296699:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830348.296702:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830348.296707:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830348.296717:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cb40 x1874672086035328/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830348.296725:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830348.296728:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787830348.296730:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830348.296733:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3783 00000100:00080000:2.0:1787830348.296737:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830348.296740:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830348.296744:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830348.296753:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498de00 x1874672086035456/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830348.296760:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830348.296764:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787830348.296765:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830348.296769:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3783 00000100:00080000:2.0:1787830348.296772:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830348.296776:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830348.296780:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:2.0:1787830348.296789:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f480 x1874672086035584/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830348.296856:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830348.296860:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787830348.296863:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830348.296867:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3783 00000100:00080000:2.0:1787830348.296871:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830348.296874:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830348.296879:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:0.0:1787830348.297054:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672086034688:192.168.203.141@tcp:250:kworker.0 00000800:00000100:0.0:1787830348.297115:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830348.297157:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830348/real 1787830348] req@ffff9e35c498fc00 x1874672086034688/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830373 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830348.297173:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498fc00 x1874672086034688/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830373 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830348.297183:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830348.297187:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830348.297191:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672086034688:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1787830348.297197:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086034816:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830348.297221:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830348.297234:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830348/real 1787830348] req@ffff9e35c498ed00 x1874672086034816/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830373 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830348.297252:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498ed00 x1874672086034816/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830373 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830348.297260:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830348.297263:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830348.297266:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086034816:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830348.297274:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086034944:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830348.297294:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830348.297306:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830348/real 1787830348] req@ffff9e35c498c000 x1874672086034944/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830373 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830348.297317:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c000 x1874672086034944/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830373 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830348.297326:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830348.297329:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830348.297331:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086034944:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830348.297337:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086035072:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830348.297356:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830348.297367:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830348/real 1787830348] req@ffff9e35c498e940 x1874672086035072/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830373 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830348.297377:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e940 x1874672086035072/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830373 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830348.297385:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830348.297388:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830348.297391:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086035072:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830348.297397:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086035200:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830348.297415:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787830348.297745:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c780 x1874672086035712/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830348.297953:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830348/real 1787830348] req@ffff9e35c498d680 x1874672086035200/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830373 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830348.297969:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498d680 x1874672086035200/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830373 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830348.297990:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830348.297993:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830348.297997:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086035200:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830348.298003:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086035328:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830348.298030:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830348.298043:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830348/real 1787830348] req@ffff9e35c498cb40 x1874672086035328/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830373 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830348.298054:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498cb40 x1874672086035328/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830373 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830348.298062:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830348.298066:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830348.298069:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086035328:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830348.298074:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086035456:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830348.298093:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830348.298104:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830348/real 1787830348] req@ffff9e35c498de00 x1874672086035456/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830373 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830348.298115:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498de00 x1874672086035456/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830373 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830348.298128:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830348.298131:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830348.298134:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086035456:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830348.298140:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086035584:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830348.298160:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830348.298170:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830348/real 1787830348] req@ffff9e35c498f480 x1874672086035584/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830373 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830348.298181:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f480 x1874672086035584/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830373 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830348.298189:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830348.298193:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830348.298196:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086035584:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830348.298233:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086035712:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830348.298254:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830348.298265:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830348/real 1787830348] req@ffff9e35c498c780 x1874672086035712/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830373 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830348.298277:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c780 x1874672086035712/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830373 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830348.298286:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830348.298289:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830348.298292:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086035712:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830348.424146:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830348.487976:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:0.0:1787830348.488394:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830348.488896:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787830349.448116:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830349.511924:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787830349.511925:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787830349.511965:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830350.471955:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830350.535906:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787830350.535920:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787830350.535957:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830351.495946:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830351.559946:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787830351.560075:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:0.0:1787830351.560138:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830352.519916:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830352.583916:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787830352.583968:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830352.584148:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00080000:0.0:1787830353.415983:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830353.415995:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787830353.415998:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830353.416009:0:27606:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3788 00000100:00080000:0.0:1787830353.416020:0:27606:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830353.416026:0:27606:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787830353.416064:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7192c0 x1874672086035840/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830353.416099:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830353.416103:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787830353.416104:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830353.416108:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3788 00000100:00080000:0.0:1787830353.416111:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830353.416115:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830353.416125:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830353.416138:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718f00 x1874672086035968/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830353.416153:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830353.416155:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787830353.416157:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830353.416160:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3788 00000100:00080000:0.0:1787830353.416163:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830353.416166:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830353.416170:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830353.416179:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a940 x1874672086036096/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830353.416187:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830353.416190:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787830353.416192:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830353.416196:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3788 00000100:00080000:0.0:1787830353.416199:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830353.416202:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830353.416207:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:0.0:1787830353.416217:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718780 x1874672086036224/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830353.416226:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830353.416229:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787830353.416230:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830353.416234:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3788 00000100:00080000:0.0:1787830353.416238:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830353.416241:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830353.416246:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:0.0:1787830353.416256:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da719a40 x1874672086036352/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830353.416264:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830353.416267:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787830353.416269:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830353.416273:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3788 00000100:00080000:0.0:1787830353.416276:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830353.416279:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830353.416284:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830353.416293:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7183c0 x1874672086036480/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830353.416301:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830353.416304:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787830353.416306:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830353.416309:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3788 00000100:00080000:0.0:1787830353.416312:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830353.416316:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830353.416319:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830353.416328:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718000 x1874672086036608/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830353.416342:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830353.416345:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787830353.416346:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830353.416350:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3788 00000100:00080000:0.0:1787830353.416353:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830353.416356:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830353.416360:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:0.0:1787830353.416370:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a580 x1874672086036736/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830353.416378:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830353.416381:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787830353.416382:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830353.416386:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3788 00000100:00080000:0.0:1787830353.416389:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830353.416392:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830353.416396:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:0.0:1787830353.416406:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718b40 x1874672086036864/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787830353.416513:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672086035840:192.168.203.141@tcp:250:kworker.0 00000800:00000100:3.0:1787830353.416671:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787830353.416711:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830353/real 1787830353] req@ffff9e35da7192c0 x1874672086035840/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830378 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787830353.416736:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7192c0 x1874672086035840/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830378 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830353.416746:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787830353.416749:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787830353.416752:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672086035840:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1787830353.416759:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086035968:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787830353.416777:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787830353.416787:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830353/real 1787830353] req@ffff9e35da718f00 x1874672086035968/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830378 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787830353.416842:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672086035968/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830378 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830353.416850:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787830353.416853:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787830353.416855:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086035968:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787830353.416861:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086036096:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787830353.416885:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787830353.416895:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830353/real 1787830353] req@ffff9e35da71a940 x1874672086036096/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830378 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787830353.416905:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672086036096/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830378 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830353.416912:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787830353.416914:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787830353.416917:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086036096:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787830353.416922:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086036224:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787830353.416935:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787830353.416942:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830353/real 1787830353] req@ffff9e35da718780 x1874672086036224/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830378 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787830353.416951:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718780 x1874672086036224/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830378 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830353.416963:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787830353.416966:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787830353.416969:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086036224:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787830353.416973:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086036352:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787830353.416988:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787830353.416997:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830353/real 1787830353] req@ffff9e35da719a40 x1874672086036352/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830378 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787830353.417006:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672086036352/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830378 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830353.417013:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787830353.417015:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787830353.417018:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086036352:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787830353.417022:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086036480:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787830353.417037:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787830353.417045:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830353/real 1787830353] req@ffff9e35da7183c0 x1874672086036480/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830378 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787830353.417054:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672086036480/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830378 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830353.417060:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787830353.417063:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787830353.417066:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086036480:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787830353.417071:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086036608:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787830353.417084:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787830353.417091:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830353/real 1787830353] req@ffff9e35da718000 x1874672086036608/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830378 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787830353.417100:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718000 x1874672086036608/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830378 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830353.417107:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787830353.417110:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787830353.417112:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086036608:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787830353.417117:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086036736:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787830353.417130:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787830353.417138:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830353/real 1787830353] req@ffff9e35da71a580 x1874672086036736/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830378 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787830353.417148:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672086036736/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830378 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830353.417155:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787830353.417158:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787830353.417160:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086036736:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787830353.417166:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086036864:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787830353.417180:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787830353.417189:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830353/real 1787830353] req@ffff9e35da718b40 x1874672086036864/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830378 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787830353.417199:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718b40 x1874672086036864/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830378 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830353.417206:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787830353.417209:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787830353.417212:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086036864:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830353.543949:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830353.608016:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787830353.608048:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830353.608762:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787830354.568014:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830354.631933:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787830354.632234:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787830354.632254:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787830355.591961:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830355.655928:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787830355.655986:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787830355.656393:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787830356.616104:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787830356.679966:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787830356.680000:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787830356.680026:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787830357.639945:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830357.703965:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787830357.704159:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787830357.704165:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787830358.535980:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830358.535989:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787830358.535993:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830358.536004:0:27606:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3793 00000100:00080000:0.0:1787830358.536015:0:27606:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830358.536022:0:27606:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787830358.536060:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a1c0 x1874672086036992/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830358.536082:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830358.536086:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787830358.536088:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830358.536092:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3793 00000100:00080000:0.0:1787830358.536096:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830358.536099:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830358.536108:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830358.536122:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718b40 x1874672086037120/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787830358.536128:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672086036992:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1787830358.536136:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830358.536141:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787830358.536143:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830358.536148:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3793 00000100:00080000:0.0:1787830358.536152:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830358.536157:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830358.536163:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830358.536175:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a580 x1874672086037248/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830358.536184:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830358.536187:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787830358.536188:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830358.536192:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3793 00000800:00000100:1.0:1787830358.536198:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787830358.536199:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830358.536206:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830358.536212:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:0.0:1787830358.536223:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718000 x1874672086037376/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830358.536232:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830358.536235:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787830358.536237:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830358.536241:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3793 00000100:00080000:0.0:1787830358.536245:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1787830358.536250:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830358/real 1787830358] req@ffff9e35da71a1c0 x1874672086036992/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830383 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830358.536251:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830358.536258:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:0.0:1787830358.536275:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7183c0 x1874672086037504/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787830358.536278:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a1c0 x1874672086036992/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830383 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830358.536299:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830358.536301:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830358.536304:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787830358.536308:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830358.536309:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830358.536317:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672086036992:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1787830358.536320:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3793 00000100:00080000:0.0:1787830358.536330:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830358.536335:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830358.536339:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830358.536346:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086037120:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830358.536352:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da719a40 x1874672086037632/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830358.536364:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830358.536367:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787830358.536369:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830358.536373:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3793 00000100:00080000:0.0:1787830358.536378:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1787830358.536378:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787830358.536385:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830358.536390:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830358.536396:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830358/real 1787830358] req@ffff9e35da718b40 x1874672086037120/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830383 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830358.536403:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718780 x1874672086037760/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830358.536432:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830358.536437:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787830358.536439:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830358.536444:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3793 00000100:00100000:1.0:1787830358.536446:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718b40 x1874672086037120/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830383 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830358.536450:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830358.536457:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:1.0:1787830358.536461:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830358.536467:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000008:00100000:0.0:1787830358.536469:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:1.0:1787830358.536473:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086037120:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787830358.536482:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086037248:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830358.536485:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a940 x1874672086037888/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830358.536497:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830358.536500:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787830358.536503:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830358.536509:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3793 00000800:00000100:1.0:1787830358.536510:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787830358.536516:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830358.536521:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830358.536526:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:1.0:1787830358.536527:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830358/real 1787830358] req@ffff9e35da71a580 x1874672086037248/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830383 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830358.536541:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718f00 x1874672086038016/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787830358.536545:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672086037248/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830383 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830358.536564:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830358.536567:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830358.536572:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086037248:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787830358.536579:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086037376:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787830358.536600:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830358.536611:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830358/real 1787830358] req@ffff9e35da718000 x1874672086037376/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830383 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830358.536622:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718000 x1874672086037376/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830383 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830358.536636:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830358.536639:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830358.536650:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086037376:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830358.536657:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086037504:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787830358.536676:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830358.536688:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830358/real 1787830358] req@ffff9e35da7183c0 x1874672086037504/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830383 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830358.536709:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672086037504/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830383 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830358.536720:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830358.536724:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830358.536727:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086037504:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830358.536747:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086037632:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787830358.536782:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830358.536821:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830358/real 1787830358] req@ffff9e35da719a40 x1874672086037632/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830383 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830358.536834:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672086037632/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830383 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830358.536843:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830358.536846:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830358.536850:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086037632:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787830358.536856:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086037760:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787830358.536876:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830358.536890:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830358/real 1787830358] req@ffff9e35da718780 x1874672086037760/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830383 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830358.536901:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718780 x1874672086037760/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830383 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830358.536908:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830358.536910:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830358.536913:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086037760:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787830358.536918:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086037888:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787830358.536935:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830358.536945:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830358/real 1787830358] req@ffff9e35da71a940 x1874672086037888/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830383 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830358.536955:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672086037888/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830383 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830358.536962:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830358.536965:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830358.536967:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086037888:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830358.536973:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086038016:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787830358.536990:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830358.537000:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830358/real 1787830358] req@ffff9e35da718f00 x1874672086038016/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830383 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830358.537011:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672086038016/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830383 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830358.537018:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830358.537021:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830358.537024:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086038016:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830358.664680:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830358.727916:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787830358.727916:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787830358.727987:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830359.688026:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830359.751923:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787830359.751975:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787830359.752117:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830360.712000:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830360.775912:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787830360.775912:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787830360.776074:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830361.735954:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830361.799927:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787830361.800014:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787830361.800350:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787830362.759927:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830362.823927:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787830362.823955:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787830362.824590:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787830363.655932:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830363.655944:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787830363.655948:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830363.655958:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3799 00000100:00080000:1.0:1787830363.655969:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830363.655976:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787830363.656011:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4780 x1874672086038144/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830363.656043:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830363.656046:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787830363.656049:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830363.656053:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3799 00000100:00080000:1.0:1787830363.656056:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830363.656059:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830363.656070:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830363.656083:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6d00 x1874672086038272/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787830363.656091:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672086038144:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787830363.656097:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830363.656101:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787830363.656103:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830363.656107:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3799 00000100:00080000:1.0:1787830363.656111:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830363.656115:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830363.656120:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830363.656130:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7840 x1874672086038400/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830363.656139:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830363.656142:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787830363.656144:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830363.656149:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3799 00000800:00000100:3.0:1787830363.656152:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787830363.656154:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830363.656160:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830363.656166:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:1.0:1787830363.656178:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f43c0 x1874672086038528/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830363.656186:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830363.656195:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787830363.656198:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1787830363.656199:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830363/real 1787830363] req@ffff9e35da6f4780 x1874672086038144/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830388 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830363.656206:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3799 00000100:00080000:1.0:1787830363.656211:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830363.656217:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830363.656221:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:3.0:1787830363.656227:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4780 x1874672086038144/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830388 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830363.656235:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f61c0 x1874672086038656/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830363.656241:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830363.656248:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830363.656249:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:1.0:1787830363.656255:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:3.0:1787830363.656257:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672086038144:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787830363.656258:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830363.656266:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3799 00000100:00080000:1.0:1787830363.656270:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830363.656274:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830363.656279:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830363.656288:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086038272:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787830363.656292:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4780 x1874672086038784/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830363.656303:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830363.656307:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787830363.656309:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830363.656313:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3799 00000100:00080000:1.0:1787830363.656317:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1787830363.656319:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787830363.656324:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830363.656336:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830363.656338:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830363/real 1787830363] req@ffff9e35da6f6d00 x1874672086038272/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830388 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830363.656349:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7480 x1874672086038912/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787830363.656355:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6d00 x1874672086038272/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830388 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830363.656363:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830363.656370:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787830363.656371:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830363.656372:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830363.656380:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:1.0:1787830363.656381:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3799 00000100:00100000:3.0:1787830363.656389:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086038272:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787830363.656390:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830363.656397:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787830363.656401:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086038400:192.168.203.141@tcp:38:kworker.0 00000008:00100000:1.0:1787830363.656403:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:1.0:1787830363.656418:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6580 x1874672086039040/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830363.656427:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:3.0:1787830363.656430:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787830363.656444:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830363/real 1787830363] req@ffff9e35da6f7840 x1874672086038400/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830388 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830363.656446:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787830363.656450:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830363.656455:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3799 00000100:00080000:1.0:1787830363.656459:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830363.656464:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830363.656472:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:3.0:1787830363.656478:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7840 x1874672086038400/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830388 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830363.656487:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4f00 x1874672086039168/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830363.656490:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787830363.656500:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787830363.656506:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086038400:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787830363.656513:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086038528:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787830363.656535:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787830363.656546:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830363/real 1787830363] req@ffff9e35da6f43c0 x1874672086038528/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830388 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787830363.656564:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f43c0 x1874672086038528/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830388 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830363.656571:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787830363.656574:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787830363.656576:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086038528:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787830363.656582:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086038656:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787830363.656599:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787830363.656609:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830363/real 1787830363] req@ffff9e35da6f61c0 x1874672086038656/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830388 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787830363.656625:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f61c0 x1874672086038656/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830388 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830363.656633:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787830363.656636:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787830363.656639:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086038656:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787830363.656658:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086038784:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787830363.656675:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787830363.656685:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830363/real 1787830363] req@ffff9e35da6f4780 x1874672086038784/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830388 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787830363.656695:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4780 x1874672086038784/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830388 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830363.656707:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787830363.656709:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787830363.656712:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086038784:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787830363.656718:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086038912:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787830363.656748:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787830363.656760:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830363/real 1787830363] req@ffff9e35da6f7480 x1874672086038912/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830388 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787830363.656769:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7480 x1874672086038912/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830388 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830363.656777:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787830363.656779:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787830363.656782:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086038912:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787830363.656787:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086039040:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787830363.656832:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787830363.656843:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830363/real 1787830363] req@ffff9e35da6f6580 x1874672086039040/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830388 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787830363.656855:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6580 x1874672086039040/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830388 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830363.656862:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787830363.656865:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787830363.656868:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086039040:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787830363.656873:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086039168:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787830363.656890:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787830363.656899:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830363/real 1787830363] req@ffff9e35da6f4f00 x1874672086039168/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830388 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787830363.656910:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4f00 x1874672086039168/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830388 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830363.656917:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787830363.656920:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787830363.656922:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086039168:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830363.784083:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830363.847935:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787830363.847935:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787830363.847983:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830364.808036:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830364.872122:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787830364.872746:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787830364.873721:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787830365.831925:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830365.895929:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787830365.896024:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830365.896651:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787830366.855935:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830366.919858:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787830366.919880:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787830366.919953:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830367.880055:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830367.943922:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787830367.943958:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787830367.944864:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00080000:0.0:1787830368.776371:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830368.776383:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787830368.776387:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830368.776397:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3804 00000100:00080000:0.0:1787830368.776406:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830368.776413:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787830368.776449:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7192c0 x1874672086039296/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787830368.776528:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672086039296:192.168.203.141@tcp:250:kworker.0 00000800:00000100:3.0:1787830368.776584:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787830368.776615:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830368.776619:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787830368.776621:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830368.776625:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3804 00000100:00080000:0.0:1787830368.776628:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830368.776632:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830368.776642:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830368.776655:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718f00 x1874672086039424/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787830368.776823:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830368/real 1787830368] req@ffff9e35da7192c0 x1874672086039296/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830393 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787830368.776840:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7192c0 x1874672086039296/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830393 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830368.776849:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787830368.776852:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787830368.776856:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672086039296:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1787830368.776878:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086039424:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787830368.776901:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787830368.776913:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830368/real 1787830368] req@ffff9e35da718f00 x1874672086039424/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830393 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787830368.776924:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672086039424/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830393 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830368.776931:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787830368.776934:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787830368.776937:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086039424:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787830368.777037:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830368.777041:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787830368.777043:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830368.777047:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3804 00000100:00080000:0.0:1787830368.777051:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830368.777055:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830368.777060:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830368.777072:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a940 x1874672086039552/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787830368.777113:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086039552:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787830368.777135:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787830368.777153:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830368.777156:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787830368.777158:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830368.777162:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3804 00000100:00080000:0.0:1787830368.777167:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830368.777170:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830368.777176:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:0.0:1787830368.777187:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718780 x1874672086039680/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787830368.777263:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830368/real 1787830368] req@ffff9e35da71a940 x1874672086039552/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830393 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787830368.777275:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672086039552/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830393 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830368.777283:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787830368.777286:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787830368.777288:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086039552:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787830368.777300:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086039680:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787830368.777353:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787830368.777366:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830368/real 1787830368] req@ffff9e35da718780 x1874672086039680/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830393 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787830368.777377:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718780 x1874672086039680/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830393 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830368.777385:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787830368.777387:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787830368.777390:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086039680:192.168.203.141@tcp:8:kworker.0 00000100:00080000:0.0:1787830368.777467:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830368.777471:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787830368.777473:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830368.777478:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3804 00000100:00080000:0.0:1787830368.777482:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830368.777486:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830368.777491:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:0.0:1787830368.777505:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da719a40 x1874672086039808/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787830368.777565:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086039808:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787830368.777589:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787830368.777610:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830368.777613:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787830368.777615:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830368.777619:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3804 00000100:00080000:0.0:1787830368.777623:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830368.777627:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830368.777632:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830368.777643:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7183c0 x1874672086039936/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787830368.777682:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830368/real 1787830368] req@ffff9e35da719a40 x1874672086039808/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830393 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787830368.777694:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672086039808/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830393 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830368.777711:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787830368.777717:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787830368.777721:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086039808:192.168.203.141@tcp:8:kworker.0 00000100:00080000:0.0:1787830368.777724:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830368.777727:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787830368.777729:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830368.777733:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3804 00000100:00100000:3.0:1787830368.777733:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086039936:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787830368.777736:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830368.777740:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830368.777744:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1787830368.777751:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830368.777755:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718000 x1874672086040064/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787830368.777761:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830368/real 1787830368] req@ffff9e35da7183c0 x1874672086039936/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830393 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830368.777765:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830368.777768:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787830368.777770:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1787830368.777772:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672086039936/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830393 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830368.777773:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3804 00000100:00080000:0.0:1787830368.777777:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830368.777780:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:3.0:1787830368.777780:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787830368.777783:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000008:00100000:0.0:1787830368.777784:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:3.0:1787830368.777786:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086039936:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830368.777793:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a580 x1874672086040192/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787830368.777795:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086040064:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787830368.777817:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830368.777820:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787830368.777822:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830368.777826:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3804 00000100:00080000:0.0:1787830368.777830:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830368.777833:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830368.777837:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:0.0:1787830368.777847:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718b40 x1874672086040320/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787830368.777933:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787830368.777944:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830368/real 1787830368] req@ffff9e35da718000 x1874672086040064/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830393 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787830368.777954:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718000 x1874672086040064/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830393 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830368.777962:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787830368.777965:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787830368.777967:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086040064:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787830368.777979:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086040192:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787830368.777997:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787830368.778008:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830368/real 1787830368] req@ffff9e35da71a580 x1874672086040192/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830393 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787830368.778019:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672086040192/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830393 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830368.778027:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787830368.778030:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787830368.778034:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086040192:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787830368.778039:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086040320:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787830368.778057:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787830368.778067:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830368/real 1787830368] req@ffff9e35da718b40 x1874672086040320/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830393 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787830368.778078:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718b40 x1874672086040320/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830393 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830368.778086:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787830368.778088:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787830368.778091:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086040320:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830368.904211:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830368.967940:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787830368.967972:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787830368.968907:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830369.927940:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787830369.991927:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787830369.991928:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787830369.991978:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830370.951946:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787830371.015977:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830371.016372:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787830371.017087:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787830371.976569:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830372.039927:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787830372.040076:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787830372.045512:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787830372.999940:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787830373.063950:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787830373.064374:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787830373.064417:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00080000:0.0:1787830373.895939:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830373.895951:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787830373.895955:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830373.895966:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3809 00000100:00080000:0.0:1787830373.895976:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830373.895983:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787830373.896020:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a1c0 x1874672086040448/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830373.896053:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830373.896057:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787830373.896059:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830373.896063:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3809 00000100:00080000:0.0:1787830373.896067:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830373.896070:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830373.896080:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830373.896094:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718b40 x1874672086040576/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830373.896103:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830373.896107:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787830373.896109:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830373.896112:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3809 00000100:00080000:0.0:1787830373.896115:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830373.896118:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830373.896122:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830373.896131:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a580 x1874672086040704/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830373.896139:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830373.896141:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787830373.896143:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830373.896146:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3809 00000100:00080000:0.0:1787830373.896149:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830373.896153:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830373.896156:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:0.0:1787830373.896163:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718000 x1874672086040832/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830373.896171:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830373.896174:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787830373.896175:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830373.896179:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3809 00000100:00080000:0.0:1787830373.896181:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830373.896185:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830373.896190:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:0.0:1787830373.896198:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7183c0 x1874672086040960/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830373.896206:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830373.896209:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787830373.896210:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830373.896214:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3809 00000100:00080000:0.0:1787830373.896217:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830373.896220:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830373.896224:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830373.896232:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da719a40 x1874672086041088/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830373.896239:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830373.896242:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787830373.896243:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830373.896246:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3809 00000100:00080000:0.0:1787830373.896249:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830373.896252:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830373.896256:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830373.896265:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718780 x1874672086041216/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830373.896273:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830373.896276:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787830373.896278:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830373.896282:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3809 00000100:00080000:0.0:1787830373.896285:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830373.896290:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830373.896295:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:0.0:1787830373.896304:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a940 x1874672086041344/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787830373.896311:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672086040448:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1787830373.896317:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830373.896321:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787830373.896322:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830373.896327:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3809 00000100:00080000:0.0:1787830373.896330:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830373.896334:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830373.896339:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:0.0:1787830373.896351:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718f00 x1874672086041472/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787830373.896388:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787830373.896503:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830373/real 1787830373] req@ffff9e35da71a1c0 x1874672086040448/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830398 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787830373.896515:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a1c0 x1874672086040448/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830398 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830373.896523:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787830373.896527:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787830373.896531:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672086040448:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1787830373.896540:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086040576:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787830373.896561:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787830373.896573:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830373/real 1787830373] req@ffff9e35da718b40 x1874672086040576/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830398 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787830373.896595:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718b40 x1874672086040576/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830398 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830373.896604:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787830373.896607:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787830373.896610:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086040576:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787830373.896615:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086040704:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787830373.896633:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787830373.896642:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830373/real 1787830373] req@ffff9e35da71a580 x1874672086040704/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830398 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787830373.896649:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672086040704/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830398 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830373.896654:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787830373.896656:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787830373.896658:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086040704:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787830373.896675:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086040832:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787830373.896694:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787830373.896705:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830373/real 1787830373] req@ffff9e35da718000 x1874672086040832/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830398 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787830373.896731:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718000 x1874672086040832/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830398 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830373.896739:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787830373.896742:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787830373.896745:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086040832:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787830373.896750:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086040960:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787830373.896768:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787830373.896784:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830373/real 1787830373] req@ffff9e35da7183c0 x1874672086040960/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830398 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787830373.896795:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672086040960/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830398 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830373.896830:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787830373.896833:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787830373.896836:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086040960:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787830373.896842:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086041088:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787830373.896861:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787830373.896872:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830373/real 1787830373] req@ffff9e35da719a40 x1874672086041088/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830398 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787830373.896882:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672086041088/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830398 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830373.896895:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787830373.896897:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787830373.896900:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086041088:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787830373.896905:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086041216:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787830373.896923:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787830373.896933:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830373/real 1787830373] req@ffff9e35da718780 x1874672086041216/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830398 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787830373.896943:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718780 x1874672086041216/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830398 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830373.896950:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787830373.896952:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787830373.896955:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086041216:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787830373.896991:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086041344:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787830373.897008:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787830373.897028:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830373/real 1787830373] req@ffff9e35da71a940 x1874672086041344/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830398 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787830373.897039:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672086041344/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830398 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830373.897047:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787830373.897051:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787830373.897054:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086041344:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787830373.897059:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086041472:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787830373.897332:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787830373.897343:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830373/real 1787830373] req@ffff9e35da718f00 x1874672086041472/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830398 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787830373.897354:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672086041472/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830398 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830373.897361:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787830373.897364:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787830373.897367:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086041472:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830374.023978:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787830374.087946:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787830374.087947:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787830374.087995:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830375.047937:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830375.111937:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787830375.112059:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787830375.112555:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830376.071934:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:1.0:1787830376.136269:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787830376.136712:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787830376.137812:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787830377.096006:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787830377.159921:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787830377.159950:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787830377.159961:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830378.119950:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787830378.183935:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787830378.183974:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830378.185855:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00080000:2.0:1787830379.015986:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830379.015998:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787830379.016002:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830379.016013:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3814 00000100:00080000:2.0:1787830379.016024:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830379.016030:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787830379.016066:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e580 x1874672086041600/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830379.016092:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830379.016096:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787830379.016098:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830379.016102:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3814 00000100:00080000:2.0:1787830379.016106:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830379.016109:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830379.016119:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830379.016133:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c3c0 x1874672086041728/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787830379.016140:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672086041600:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1787830379.016146:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830379.016151:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787830379.016153:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830379.016157:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3814 00000100:00080000:2.0:1787830379.016161:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830379.016164:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830379.016169:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830379.016186:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f840 x1874672086041856/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830379.016200:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:3.0:1787830379.016205:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1787830379.016206:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787830379.016210:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830379.016215:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3814 00000100:00080000:2.0:1787830379.016219:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830379.016223:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830379.016228:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:2.0:1787830379.016240:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f0c0 x1874672086041984/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830379.016249:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1787830379.016253:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830379/real 1787830379] req@ffff9e35c498e580 x1874672086041600/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830404 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830379.016254:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787830379.016258:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830379.016266:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3814 00000100:00080000:2.0:1787830379.016271:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830379.016275:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787830379.016279:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e580 x1874672086041600/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830404 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1787830379.016281:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00080000:3.0:1787830379.016295:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:2.0:1787830379.016299:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cf00 x1874672086042112/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830379.016302:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787830379.016311:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672086041600:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1787830379.016314:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830379.016319:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787830379.016321:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830379.016325:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3814 00000100:00080000:2.0:1787830379.016329:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830379.016332:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830379.016337:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830379.016340:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086041728:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787830379.016349:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e1c0 x1874672086042240/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830379.016358:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830379.016361:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787830379.016363:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:3.0:1787830379.016367:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1787830379.016371:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3814 00000100:00080000:2.0:1787830379.016376:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830379.016380:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830379.016385:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830379.016385:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830379/real 1787830379] req@ffff9e35c498c3c0 x1874672086041728/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830404 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830379.016398:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e580 x1874672086042368/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787830379.016402:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c3c0 x1874672086041728/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830404 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830379.016420:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830379.016422:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830379.016426:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787830379.016429:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:2.0:1787830379.016431:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1787830379.016435:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086041728:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1787830379.016440:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3814 00000100:00080000:2.0:1787830379.016445:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1787830379.016446:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086041856:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1787830379.016453:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830379.016460:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000800:00000100:3.0:1787830379.016470:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787830379.016475:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c780 x1874672086042496/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787830379.016483:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830379/real 1787830379] req@ffff9e35c498f840 x1874672086041856/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830404 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830379.016510:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1787830379.016513:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f840 x1874672086041856/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830404 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830379.016515:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787830379.016519:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830379.016529:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3814 00000100:00080000:3.0:1787830379.016530:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830379.016534:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830379.016537:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:2.0:1787830379.016542:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787830379.016544:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086041856:192.168.203.141@tcp:38:kworker.0 00000008:00100000:2.0:1787830379.016549:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:3.0:1787830379.016554:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086041984:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787830379.016565:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f480 x1874672086042624/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787830379.016576:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787830379.016588:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830379/real 1787830379] req@ffff9e35c498f0c0 x1874672086041984/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830404 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787830379.016600:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f0c0 x1874672086041984/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830404 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830379.016615:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787830379.016618:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787830379.016621:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086041984:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787830379.016626:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086042112:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787830379.016656:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787830379.016666:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830379/real 1787830379] req@ffff9e35c498cf00 x1874672086042112/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830404 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787830379.016676:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498cf00 x1874672086042112/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830404 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830379.016685:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787830379.016688:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787830379.016690:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086042112:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787830379.016716:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086042240:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787830379.016736:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787830379.016747:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830379/real 1787830379] req@ffff9e35c498e1c0 x1874672086042240/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830404 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787830379.016764:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e1c0 x1874672086042240/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830404 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830379.016791:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787830379.016795:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787830379.016826:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086042240:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787830379.016832:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086042368:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1787830379.016852:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787830379.016864:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830379/real 1787830379] req@ffff9e35c498e580 x1874672086042368/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830404 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787830379.016876:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e580 x1874672086042368/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830404 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830379.016884:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787830379.016887:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787830379.016890:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086042368:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787830379.016895:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086042496:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787830379.016911:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787830379.016919:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830379/real 1787830379] req@ffff9e35c498c780 x1874672086042496/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830404 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787830379.016929:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c780 x1874672086042496/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830404 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830379.016937:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787830379.016940:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787830379.016943:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086042496:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787830379.016948:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086042624:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787830379.016979:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787830379.016990:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830379/real 1787830379] req@ffff9e35c498f480 x1874672086042624/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830404 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787830379.017001:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f480 x1874672086042624/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830404 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830379.017009:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787830379.017011:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787830379.017014:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086042624:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830379.143910:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830379.207928:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787830379.208002:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787830379.208824:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830380.168953:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830380.231881:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787830380.232080:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830380.232196:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787830381.192018:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830381.255931:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787830381.255988:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787830381.256459:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830382.216742:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830382.280085:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787830382.280200:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787830382.280358:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787830383.239950:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787830383.303933:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787830383.303936:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787830383.304023:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787830384.136037:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830384.136049:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787830384.136052:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830384.136063:0:27606:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3819 00000100:00080000:0.0:1787830384.136074:0:27606:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830384.136082:0:27606:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787830384.136118:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7192c0 x1874672086042752/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830384.136335:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830384.136339:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787830384.136340:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830384.136345:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3819 00000100:00080000:0.0:1787830384.136349:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830384.136353:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830384.136362:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830384.136377:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718f00 x1874672086042880/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830384.136387:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830384.136390:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787830384.136392:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830384.136395:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3819 00000100:00080000:0.0:1787830384.136398:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830384.136402:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830384.136406:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830384.136417:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a940 x1874672086043008/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787830384.136520:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672086042752:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1787830384.136745:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086042880:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787830384.136846:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830384.136849:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787830384.136851:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830384.136859:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3819 00000100:00080000:0.0:1787830384.136863:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830384.136867:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830384.136872:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:3.0:1787830384.136880:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086043008:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830384.136883:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718780 x1874672086043136/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830384.136892:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830384.136896:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787830384.136898:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830384.136901:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3819 00000100:00080000:0.0:1787830384.136904:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830384.136907:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830384.136912:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:0.0:1787830384.136921:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da719a40 x1874672086043264/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787830384.136921:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086043136:192.168.203.141@tcp:8:kworker.0 00000100:00080000:0.0:1787830384.136942:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1787830384.136944:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086043264:192.168.203.141@tcp:8:kworker.0 00000100:00080000:0.0:1787830384.136946:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787830384.136948:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830384.136951:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3819 00000100:00080000:0.0:1787830384.136955:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830384.136958:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830384.136962:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830384.136972:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7183c0 x1874672086043392/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830384.136988:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830384.136992:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787830384.136993:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830384.136997:0:27606:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3819 00000100:00080000:0.0:1787830384.137001:0:27606:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830384.137005:0:27606:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830384.137009:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830384.137019:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718000 x1874672086043520/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830384.137027:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830384.137030:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787830384.137032:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830384.137035:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3819 00000100:00080000:0.0:1787830384.137039:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830384.137042:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830384.137047:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:0.0:1787830384.137056:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a580 x1874672086043648/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787830384.137064:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086043392:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787830384.137070:0:27606:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830384.137073:0:27606:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787830384.137074:0:27606:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830384.137079:0:27606:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3819 00000100:00080000:0.0:1787830384.137081:0:27606:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830384.137084:0:27606:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830384.137088:0:27606:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:3.0:1787830384.137089:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086043520:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830384.137097:0:27606:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718b40 x1874672086043776/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787830384.137110:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086043648:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1787830384.137130:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086043776:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830384.264044:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787830384.328264:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830384.328336:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787830384.329510:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787830385.288120:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830385.351921:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787830385.351981:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787830385.352138:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830386.311937:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787830386.376176:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787830386.376531:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787830386.376554:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000400:00000100:0.0:1787830387.209938:0:2397:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.141:988 00000400:00000100:0.0:1787830387.210003:0:2397:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.141@tcp at host 192.168.203.141 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1787830387.210023:0:2397:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:1.0:1787830387.210227:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830384/real 1787830387] req@ffff9e35da7192c0 x1874672086042752/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830409 ref 2 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830387.210252:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7192c0 x1874672086042752/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830409 ref 2 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830387.210330:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830387.210336:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830387.210341:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672086042752:192.168.203.141@tcp:250:kworker.0 00000800:00000100:0.0:1787830387.210685:0:2397:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000800:00000100:0.0:1787830387.210708:0:2397:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:0.0:1787830387.210716:0:2397:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:0.0:1787830387.210724:0:2397:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:0.0:1787830387.210730:0:2397:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:0.0:1787830387.210737:0:2397:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:0.0:1787830387.210744:0:2397:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:0.0:1787830387.210751:0:2397:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:0.0:1787830387.210770:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830384/real 1787830387] req@ffff9e35da71a580 x1874672086043648/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830409 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830387.210784:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672086043648/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830409 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830387.210794:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830387.211441:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830387.211450:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086043648:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830387.211460:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830384/real 1787830387] req@ffff9e35da718b40 x1874672086043776/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830409 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830387.211476:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718b40 x1874672086043776/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830409 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830387.211484:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830387.211487:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830387.211490:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086043776:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830387.211495:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830384/real 1787830387] req@ffff9e35da7183c0 x1874672086043392/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830409 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830387.211506:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672086043392/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830409 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830387.211512:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830387.211515:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830387.211518:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086043392:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830387.211523:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830384/real 1787830387] req@ffff9e35da718000 x1874672086043520/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830409 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830387.211533:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718000 x1874672086043520/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830409 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830387.211540:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830387.211543:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830387.211546:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086043520:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830387.211551:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830384/real 1787830387] req@ffff9e35da719a40 x1874672086043264/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830409 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830387.211561:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672086043264/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830409 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830387.211569:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830387.211582:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830387.211586:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086043264:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830387.211591:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830384/real 1787830387] req@ffff9e35da718780 x1874672086043136/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830409 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830387.211603:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718780 x1874672086043136/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830409 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830387.211611:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830387.211614:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830387.211617:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086043136:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830387.211624:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830384/real 1787830387] req@ffff9e35da71a940 x1874672086043008/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830409 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830387.211635:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672086043008/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830409 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830387.211643:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830387.211647:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830387.211650:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086043008:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830387.211656:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830384/real 1787830387] req@ffff9e35da718f00 x1874672086042880/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830409 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830387.211667:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672086042880/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830409 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830387.211675:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830387.211678:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830387.211681:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086042880:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787830387.336175:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830387.399949:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787830387.400027:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787830387.400112:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830388.359939:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830388.423932:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787830388.424020:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787830388.424771:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787830389.257673:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830389.257687:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787830389.257691:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830389.257702:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3824 00000100:00080000:2.0:1787830389.257715:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830389.257723:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787830389.257776:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498de00 x1874672086043904/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830389.257832:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830389.257837:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787830389.257839:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830389.257844:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3824 00000100:00080000:2.0:1787830389.257848:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830389.257851:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830389.257863:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830389.257875:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cb40 x1874672086044032/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830389.257884:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830389.257887:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787830389.257889:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830389.257970:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3824 00000100:00080000:2.0:1787830389.257975:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830389.257978:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830389.257983:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830389.257995:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498d680 x1874672086044160/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830389.258003:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830389.258005:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787830389.258007:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830389.258011:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3824 00000100:00080000:2.0:1787830389.258014:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830389.258016:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830389.258021:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:2.0:1787830389.258030:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e940 x1874672086044288/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830389.258037:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830389.258039:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787830389.258040:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830389.258043:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3824 00000100:00080000:2.0:1787830389.258046:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830389.258048:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830389.258051:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:2.0:1787830389.258140:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c000 x1874672086044416/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830389.258150:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830389.258153:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787830389.258154:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830389.258158:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3824 00000100:00080000:2.0:1787830389.258162:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830389.258166:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830389.258170:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830389.258180:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498ed00 x1874672086044544/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830389.258188:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830389.258191:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787830389.258194:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830389.258197:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3824 00000100:00080000:2.0:1787830389.258201:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830389.258206:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830389.258210:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830389.258219:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498fc00 x1874672086044672/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830389.258227:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830389.258230:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787830389.258231:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830389.258235:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3824 00000100:00080000:2.0:1787830389.258238:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830389.258242:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830389.258246:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:2.0:1787830389.258256:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f480 x1874672086044800/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830389.258264:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830389.258266:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787830389.258268:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830389.258272:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3824 00000100:00080000:2.0:1787830389.258275:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830389.258278:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830389.258282:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:2.0:1787830389.258292:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c780 x1874672086044928/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830389.258416:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672086043904:192.168.203.141@tcp:250:kworker.0 00000800:00000100:0.0:1787830389.258685:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830389.258730:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830389/real 1787830389] req@ffff9e35c498de00 x1874672086043904/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830414 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830389.258747:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498de00 x1874672086043904/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830414 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830389.258756:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830389.258760:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830389.258764:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672086043904:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1787830389.258772:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086044032:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830389.258796:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830389.258835:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830389/real 1787830389] req@ffff9e35c498cb40 x1874672086044032/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830414 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830389.258847:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498cb40 x1874672086044032/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830414 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830389.258855:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830389.258858:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830389.258862:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086044032:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830389.258868:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086044160:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830389.258886:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830389.258896:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830389/real 1787830389] req@ffff9e35c498d680 x1874672086044160/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830414 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830389.258907:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498d680 x1874672086044160/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830414 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830389.258914:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830389.258917:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830389.258920:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086044160:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830389.258926:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086044288:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830389.258944:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830389.258954:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830389/real 1787830389] req@ffff9e35c498e940 x1874672086044288/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830414 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830389.258971:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e940 x1874672086044288/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830414 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830389.258984:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830389.258988:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830389.258991:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086044288:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830389.258996:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086044416:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830389.259012:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830389.259020:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830389/real 1787830389] req@ffff9e35c498c000 x1874672086044416/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830414 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830389.261390:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c000 x1874672086044416/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830414 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830389.261405:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830389.261410:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830389.261414:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086044416:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830389.261880:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086044544:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830389.261944:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830389.261967:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830389/real 1787830389] req@ffff9e35c498ed00 x1874672086044544/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830414 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830389.261980:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498ed00 x1874672086044544/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830414 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830389.261995:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830389.261998:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830389.262001:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086044544:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830389.262008:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086044672:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830389.262023:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830389.262130:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830389/real 1787830389] req@ffff9e35c498fc00 x1874672086044672/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830414 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830389.262143:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498fc00 x1874672086044672/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830414 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830389.262150:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830389.262153:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830389.262156:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086044672:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830389.262162:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086044800:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830389.262184:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830389.262194:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830389/real 1787830389] req@ffff9e35c498f480 x1874672086044800/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830414 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830389.262204:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f480 x1874672086044800/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830414 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830389.262212:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830389.262215:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830389.262218:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086044800:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830389.262225:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086044928:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830389.262242:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830389.262251:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830389/real 1787830389] req@ffff9e35c498c780 x1874672086044928/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830414 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830389.262260:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c780 x1874672086044928/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830414 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830389.262267:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830389.262269:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830389.262272:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086044928:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830389.384565:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787830389.447965:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830389.448014:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787830389.448389:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787830390.407941:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830390.471947:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787830390.471947:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787830390.472206:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830391.432001:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:3.0:1787830391.496265:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787830391.496295:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787830391.496318:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787830392.455943:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787830392.519930:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787830392.519950:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787830392.520012:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830393.480127:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787830393.544028:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787830393.544232:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787830393.544279:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00080000:3.0:1787830394.376365:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830394.376374:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787830394.376377:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830394.376388:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3829 00000100:00080000:3.0:1787830394.376398:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830394.376405:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787830394.376857:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498b40 x1874672086045056/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830394.376924:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672086045056:192.168.203.141@tcp:250:kworker.0 00000800:00000100:0.0:1787830394.376983:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1787830394.377061:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830394.377066:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787830394.377068:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830394.377073:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3829 00000100:00080000:3.0:1787830394.377087:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830394.377091:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830394.377101:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830394.377158:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498f00 x1874672086045184/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830394.377402:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830394/real 1787830394] req@ffff9e35c6498b40 x1874672086045056/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830419 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830394.377419:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498b40 x1874672086045056/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830419 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830394.377429:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830394.377432:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830394.377436:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672086045056:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1787830394.377457:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086045184:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830394.377483:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830394.377497:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830394/real 1787830394] req@ffff9e35c6498f00 x1874672086045184/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830419 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830394.377511:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498f00 x1874672086045184/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830419 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830394.377533:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830394.377537:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830394.377541:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086045184:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1787830394.377643:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830394.377647:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787830394.377649:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830394.377653:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3829 00000100:00080000:3.0:1787830394.377658:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830394.377661:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830394.377667:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830394.377747:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499e00 x1874672086045312/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830394.377791:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086045312:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830394.377835:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1787830394.377881:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830394.377886:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787830394.377888:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830394.377892:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3829 00000100:00080000:3.0:1787830394.377896:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830394.377900:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830394.377906:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:3.0:1787830394.377919:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a580 x1874672086045440/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830394.378192:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830394/real 1787830394] req@ffff9e35c6499e00 x1874672086045312/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830419 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830394.378904:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499e00 x1874672086045312/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830419 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830394.378915:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830394.378919:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830394.378922:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086045312:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830394.378942:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086045440:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830394.378971:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830394.378985:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830394/real 1787830394] req@ffff9e35c649a580 x1874672086045440/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830419 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830394.379003:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a580 x1874672086045440/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830419 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830394.379012:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830394.379015:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830394.379018:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086045440:192.168.203.141@tcp:8:kworker.0 00000100:00080000:3.0:1787830394.380048:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830394.380055:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787830394.380057:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830394.380065:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3829 00000100:00080000:3.0:1787830394.380070:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830394.380163:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830394.380174:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:3.0:1787830394.380196:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b0c0 x1874672086045568/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830394.380325:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830394.380329:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787830394.380330:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830394.380335:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3829 00000100:00080000:3.0:1787830394.380339:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830394.380343:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830394.380348:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830394.380360:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a940 x1874672086045696/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830394.380368:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830394.380371:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787830394.380372:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830394.380375:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3829 00000100:00080000:3.0:1787830394.380378:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830394.380382:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830394.380385:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830394.380394:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499680 x1874672086045824/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830394.380403:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830394.380406:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787830394.380408:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830394.380412:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3829 00000100:00080000:3.0:1787830394.380415:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830394.380418:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830394.380422:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:3.0:1787830394.380431:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498780 x1874672086045952/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830394.380439:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830394.380442:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787830394.380443:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830394.380447:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3829 00000100:00080000:3.0:1787830394.380450:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830394.380454:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830394.380458:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:3.0:1787830394.380467:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649bc00 x1874672086046080/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830394.380547:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086045568:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830394.380582:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830394.380599:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830394/real 1787830394] req@ffff9e35c649b0c0 x1874672086045568/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830419 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830394.380612:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b0c0 x1874672086045568/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830419 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830394.380621:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830394.380624:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830394.380627:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086045568:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830394.380633:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086045696:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830394.380652:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830394.380663:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830394/real 1787830394] req@ffff9e35c649a940 x1874672086045696/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830419 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830394.380746:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a940 x1874672086045696/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830419 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830394.380761:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830394.380765:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830394.380768:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086045696:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830394.380774:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086045824:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830394.380795:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830394.381514:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830394/real 1787830394] req@ffff9e35c6499680 x1874672086045824/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830419 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830394.381548:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499680 x1874672086045824/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830419 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830394.381560:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830394.381564:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830394.381567:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086045824:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830394.381574:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086045952:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830394.381595:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830394.381604:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830394/real 1787830394] req@ffff9e35c6498780 x1874672086045952/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830419 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830394.381613:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498780 x1874672086045952/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830419 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830394.381619:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830394.381621:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830394.381623:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086045952:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830394.381627:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086046080:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830394.381649:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830394.381659:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830394/real 1787830394] req@ffff9e35c649bc00 x1874672086046080/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830419 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830394.381671:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649bc00 x1874672086046080/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830419 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830394.381692:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830394.381695:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830394.381698:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086046080:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830394.504834:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787830394.567943:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:0.0:1787830394.568073:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830394.568140:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787830395.528357:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787830395.591951:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787830395.592012:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830395.592779:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787830396.551948:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830396.615939:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787830396.616002:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787830396.616006:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830397.575947:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:3.0:1787830397.639936:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787830397.641782:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787830397.643891:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830398.599932:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787830398.664031:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787830398.664042:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787830398.664402:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787830399.495904:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830399.495914:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787830399.495918:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830399.495928:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3834 00000100:00080000:1.0:1787830399.495938:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830399.495945:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787830399.495978:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4000 x1874672086046208/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830399.496007:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830399.496010:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787830399.496012:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830399.496016:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3834 00000100:00080000:1.0:1787830399.496019:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830399.496022:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830399.496033:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830399.496044:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5a40 x1874672086046336/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830399.496055:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787830399.496057:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672086046208:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787830399.496059:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787830399.496063:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830399.496068:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3834 00000100:00080000:1.0:1787830399.496070:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830399.496074:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830399.496078:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830399.496087:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5e00 x1874672086046464/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830399.496097:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830399.496100:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787830399.496101:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830399.496105:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3834 00000100:00080000:1.0:1787830399.496108:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830399.496112:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830399.496116:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000800:00000100:0.0:1787830399.496121:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830399.496129:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4b40 x1874672086046592/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830399.496138:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830399.496141:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787830399.496143:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830399.496147:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3834 00000100:00080000:1.0:1787830399.496151:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830399.496154:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830399.496158:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:0.0:1787830399.496165:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830399/real 1787830399] req@ffff9e35da6f4000 x1874672086046208/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830424 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830399.496169:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f70c0 x1874672086046720/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830399.496192:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830399.496195:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787830399.496197:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787830399.496200:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4000 x1874672086046208/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830424 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830399.496203:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3834 00000100:00080000:1.0:1787830399.496206:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830399.496210:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830399.496214:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830399.496227:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6940 x1874672086046848/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830399.496228:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830399.496236:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830399.496244:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672086046208:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787830399.496245:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830399.496253:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787830399.496255:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830399.496260:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3834 00000100:00080000:1.0:1787830399.496263:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830399.496267:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830399.496272:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830399.496273:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086046336:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787830399.496285:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4000 x1874672086046976/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830399.496296:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830399.496301:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000100:0.0:1787830399.496302:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787830399.496304:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787830399.496319:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830399/real 1787830399] req@ffff9e35da6f5a40 x1874672086046336/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830424 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830399.496323:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3834 00000100:00080000:1.0:1787830399.496331:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787830399.496338:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5a40 x1874672086046336/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830424 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830399.496338:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830399.496346:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00080000:0.0:1787830399.496353:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1787830399.496361:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4f00 x1874672086047104/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830399.496369:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830399.496373:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787830399.496374:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830399.496378:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3834 00000100:00080000:1.0:1787830399.496381:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830399.496384:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830399.496388:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:1.0:1787830399.496397:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6580 x1874672086047232/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830399.496446:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830399.496451:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086046336:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830399.496458:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086046464:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830399.496480:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830399.496503:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830399/real 1787830399] req@ffff9e35da6f5e00 x1874672086046464/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830424 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830399.496514:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5e00 x1874672086046464/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830424 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830399.496522:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830399.496525:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830399.496528:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086046464:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830399.496534:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086046592:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830399.496552:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830399.496562:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830399/real 1787830399] req@ffff9e35da6f4b40 x1874672086046592/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830424 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830399.496577:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4b40 x1874672086046592/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830424 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830399.496584:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830399.496586:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830399.496589:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086046592:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830399.496594:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086046720:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830399.496610:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830399.496621:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830399/real 1787830399] req@ffff9e35da6f70c0 x1874672086046720/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830424 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830399.496631:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f70c0 x1874672086046720/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830424 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830399.496638:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830399.496640:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830399.496643:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086046720:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830399.496647:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086046848:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830399.496664:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830399.496674:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830399/real 1787830399] req@ffff9e35da6f6940 x1874672086046848/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830424 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830399.496684:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6940 x1874672086046848/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830424 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830399.496696:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830399.496698:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830399.496701:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086046848:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830399.496723:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086046976:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830399.496742:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830399.496752:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830399/real 1787830399] req@ffff9e35da6f4000 x1874672086046976/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830424 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830399.496763:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4000 x1874672086046976/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830424 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830399.496771:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830399.496774:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830399.496777:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086046976:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830399.496782:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086047104:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830399.496842:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830399.496853:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830399/real 1787830399] req@ffff9e35da6f4f00 x1874672086047104/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830424 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830399.496864:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4f00 x1874672086047104/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830424 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830399.496871:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830399.496874:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830399.496877:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086047104:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830399.496882:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086047232:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830399.496911:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830399.496921:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830399/real 1787830399] req@ffff9e35da6f6580 x1874672086047232/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830424 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830399.496932:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6580 x1874672086047232/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830424 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830399.496939:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830399.496942:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830399.496945:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086047232:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830399.623945:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787830399.687918:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787830399.687984:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830399.688565:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787830400.648230:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830400.711886:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787830400.712129:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:0.0:1787830400.712204:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830401.671935:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787830401.735937:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787830401.736052:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830401.736069:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787830402.695934:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830402.760026:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787830402.760726:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787830402.760889:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830403.720118:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787830403.783921:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787830403.783932:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787830403.784067:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787830404.616120:0:27971:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830404.616131:0:27971:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787830404.616135:0:27971:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830404.616146:0:27971:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3839 00000100:00080000:1.0:1787830404.616156:0:27971:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830404.616163:0:27971:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787830404.616198:0:27971:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7480 x1874672086047360/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830404.616237:0:27971:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830404.616241:0:27971:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787830404.616243:0:27971:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830404.616247:0:27971:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3839 00000100:00080000:1.0:1787830404.616251:0:27971:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830404.616254:0:27971:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830404.616264:0:27971:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830404.616277:0:27971:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4780 x1874672086047488/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830404.616286:0:27971:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830404.616289:0:27971:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787830404.616291:0:27971:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830404.616294:0:27971:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3839 00000100:00080000:1.0:1787830404.616298:0:27971:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830404.616302:0:27971:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830404.616306:0:27971:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830404.616317:0:27971:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f61c0 x1874672086047616/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830404.616325:0:27971:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830404.616328:0:27971:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787830404.616330:0:27971:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830404.616333:0:27971:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3839 00000100:00080000:1.0:1787830404.616337:0:27971:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830404.616340:0:27971:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830404.616344:0:27971:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:1.0:1787830404.616354:0:27971:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f43c0 x1874672086047744/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830404.616362:0:27971:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830404.616365:0:27971:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787830404.616367:0:27971:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830404.616370:0:27971:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3839 00000100:00080000:1.0:1787830404.616374:0:27971:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830404.616377:0:27971:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830404.616381:0:27971:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:1.0:1787830404.616390:0:27971:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7840 x1874672086047872/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830404.616404:0:27971:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830404.616407:0:27971:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787830404.616408:0:27971:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830404.616412:0:27971:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3839 00000100:00080000:1.0:1787830404.616415:0:27971:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830404.616418:0:27971:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830404.616422:0:27971:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830404.616434:0:27971:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6d00 x1874672086048000/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830404.616442:0:27971:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830404.616445:0:27971:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787830404.616447:0:27971:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830404.616460:0:27971:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3839 00000100:00080000:1.0:1787830404.616464:0:27971:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830404.616468:0:27971:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830404.616472:0:27971:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830404.616482:0:27971:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6580 x1874672086048128/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830404.616491:0:27971:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830404.616494:0:27971:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787830404.616495:0:27971:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830404.616499:0:27971:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3839 00000100:00080000:1.0:1787830404.616503:0:27971:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830404.616506:0:27971:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830404.616511:0:27971:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:1.0:1787830404.616520:0:27971:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4f00 x1874672086048256/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830404.616529:0:27971:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830404.616532:0:27971:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787830404.616533:0:27971:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830404.616537:0:27971:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3839 00000100:00080000:1.0:1787830404.616540:0:27971:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830404.616543:0:27971:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830404.616547:0:27971:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:1.0:1787830404.616557:0:27971:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4000 x1874672086048384/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830404.616661:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672086047360:192.168.203.141@tcp:250:kworker.0 00000800:00000100:0.0:1787830404.616717:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830404.616758:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830404/real 1787830404] req@ffff9e35da6f7480 x1874672086047360/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830429 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830404.616773:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7480 x1874672086047360/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830429 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830404.616783:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830404.616786:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830404.616790:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672086047360:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1787830404.616820:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086047488:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830404.616844:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830404.616856:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830404/real 1787830404] req@ffff9e35da6f4780 x1874672086047488/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830429 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830404.616868:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4780 x1874672086047488/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830429 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830404.616877:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830404.616880:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830404.616883:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086047488:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830404.616889:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086047616:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830404.616907:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830404.616917:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830404/real 1787830404] req@ffff9e35da6f61c0 x1874672086047616/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830429 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830404.616927:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f61c0 x1874672086047616/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830429 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830404.616934:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830404.616937:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830404.616940:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086047616:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830404.616946:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086047744:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830404.616964:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830404.616975:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830404/real 1787830404] req@ffff9e35da6f43c0 x1874672086047744/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830429 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830404.616986:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f43c0 x1874672086047744/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830429 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830404.616994:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830404.616996:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830404.617000:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086047744:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830404.617005:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086047872:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830404.617022:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830404.617030:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830404/real 1787830404] req@ffff9e35da6f7840 x1874672086047872/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830429 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830404.617037:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7840 x1874672086047872/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830429 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830404.617042:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830404.617044:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830404.617045:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086047872:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830404.617048:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086048000:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830404.617058:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830404.617063:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830404/real 1787830404] req@ffff9e35da6f6d00 x1874672086048000/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830429 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830404.617070:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6d00 x1874672086048000/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830429 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830404.617078:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830404.617081:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830404.617084:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086048000:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830404.617088:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086048128:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830404.617104:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830404.617114:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830404/real 1787830404] req@ffff9e35da6f6580 x1874672086048128/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830429 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830404.617125:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6580 x1874672086048128/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830429 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830404.617133:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830404.617135:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830404.617138:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086048128:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830404.617143:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086048256:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830404.617157:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830404.617163:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830404/real 1787830404] req@ffff9e35da6f4f00 x1874672086048256/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830429 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830404.617170:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4f00 x1874672086048256/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830429 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830404.617175:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830404.617177:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830404.617179:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086048256:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830404.617182:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086048384:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830404.617190:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830404.617195:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830404/real 1787830404] req@ffff9e35da6f4000 x1874672086048384/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830429 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830404.617201:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4000 x1874672086048384/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830429 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830404.617205:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830404.617207:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830404.617208:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086048384:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830404.745157:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787830404.807913:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787830404.807959:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830404.808535:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787830405.768198:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830405.831928:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787830405.831928:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787830405.831978:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830406.791934:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830406.855961:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787830406.855971:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787830406.856698:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787830407.819118:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830407.879958:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787830407.880218:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787830407.880242:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830408.840028:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830408.903932:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787830408.903932:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787830408.904050:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787830409.736694:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830409.736704:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787830409.736708:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830409.736718:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3845 00000100:00080000:0.0:1787830409.736728:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830409.736741:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787830409.736773:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718f00 x1874672086048512/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830409.736851:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830409.736856:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787830409.736858:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830409.736862:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3845 00000100:00080000:0.0:1787830409.736865:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830409.736869:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830409.736878:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830409.736889:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a940 x1874672086048640/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830409.736898:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830409.736901:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787830409.736902:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830409.736906:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3845 00000100:00080000:0.0:1787830409.736909:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830409.736911:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830409.736915:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830409.736921:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718780 x1874672086048768/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830409.736929:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830409.736931:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787830409.736933:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830409.736936:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3845 00000100:00080000:0.0:1787830409.736939:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830409.736941:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830409.736945:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:0.0:1787830409.736952:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da719a40 x1874672086048896/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830409.736960:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830409.736962:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787830409.736964:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830409.736967:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3845 00000100:00080000:0.0:1787830409.736970:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830409.736973:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830409.736976:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:0.0:1787830409.736983:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718000 x1874672086049024/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830409.736991:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830409.736994:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787830409.736995:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830409.736999:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3845 00000100:00080000:0.0:1787830409.737002:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830409.737005:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830409.737008:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830409.737017:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7183c0 x1874672086049152/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830409.737025:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830409.737028:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787830409.737029:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830409.737033:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3845 00000100:00080000:0.0:1787830409.737035:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830409.737038:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830409.737042:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830409.737049:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718b40 x1874672086049280/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830409.737056:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830409.737058:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787830409.737060:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830409.737063:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3845 00000100:00080000:0.0:1787830409.737066:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830409.737068:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830409.737072:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:0.0:1787830409.737078:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a580 x1874672086049408/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830409.737085:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830409.737087:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787830409.737089:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830409.737091:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3845 00000100:00080000:0.0:1787830409.737094:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830409.737097:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830409.737100:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:0.0:1787830409.737107:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7192c0 x1874672086049536/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787830409.737271:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672086048512:192.168.203.141@tcp:250:kworker.0 00000800:00000100:1.0:1787830409.737323:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830409.737358:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830409/real 1787830409] req@ffff9e35da718f00 x1874672086048512/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830434 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830409.737374:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672086048512/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830434 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830409.737383:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830409.737386:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830409.737390:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672086048512:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1787830409.737396:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086048640:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787830409.737411:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830409.737440:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830409/real 1787830409] req@ffff9e35da71a940 x1874672086048640/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830434 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830409.737450:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672086048640/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830434 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830409.737458:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830409.737461:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830409.737463:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086048640:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787830409.737468:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086048768:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787830409.737486:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830409.737497:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830409/real 1787830409] req@ffff9e35da718780 x1874672086048768/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830434 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830409.737506:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718780 x1874672086048768/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830434 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830409.737517:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830409.737520:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830409.737523:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086048768:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787830409.737527:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086048896:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787830409.737545:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830409.737555:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830409/real 1787830409] req@ffff9e35da719a40 x1874672086048896/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830434 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830409.737565:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672086048896/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830434 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830409.737573:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830409.737576:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830409.737579:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086048896:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830409.737584:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086049024:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787830409.737601:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830409.737615:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830409/real 1787830409] req@ffff9e35da718000 x1874672086049024/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830434 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830409.737627:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718000 x1874672086049024/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830434 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830409.737635:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830409.737638:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830409.737641:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086049024:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830409.737647:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086049152:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787830409.737666:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830409.737677:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830409/real 1787830409] req@ffff9e35da7183c0 x1874672086049152/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830434 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830409.737687:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672086049152/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830434 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830409.737694:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830409.737697:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830409.737700:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086049152:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787830409.737705:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086049280:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787830409.737721:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830409.737732:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830409/real 1787830409] req@ffff9e35da718b40 x1874672086049280/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830434 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830409.737742:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718b40 x1874672086049280/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830434 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830409.737749:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830409.737751:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830409.737754:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086049280:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787830409.737760:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086049408:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787830409.737776:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830409.737787:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830409/real 1787830409] req@ffff9e35da71a580 x1874672086049408/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830434 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830409.737827:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672086049408/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830434 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830409.737837:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830409.737839:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830409.737841:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086049408:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830409.737846:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086049536:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787830409.737863:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830409.737873:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830409/real 1787830409] req@ffff9e35da7192c0 x1874672086049536/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830434 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830409.737883:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7192c0 x1874672086049536/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830434 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830409.737889:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830409.737892:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830409.737895:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086049536:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830409.865212:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787830409.927926:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787830409.927962:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830409.928117:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787830410.888640:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830410.951937:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787830410.951989:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787830410.952117:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787830411.912352:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830411.975924:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787830411.975924:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787830411.976381:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830412.937865:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787830412.999954:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787830413.000371:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787830413.001203:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830413.959900:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830414.024023:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787830414.024146:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787830414.024767:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00080000:2.0:1787830414.856036:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830414.856045:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787830414.856048:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830414.856056:0:479:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3850 00000100:00080000:2.0:1787830414.856064:0:479:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830414.856069:0:479:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787830414.856107:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e580 x1874672086049664/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830414.856131:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830414.856134:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787830414.856136:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830414.856140:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3850 00000100:00080000:2.0:1787830414.856143:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830414.856147:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830414.856156:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830414.856168:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e1c0 x1874672086049792/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830414.856176:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830414.856179:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787830414.856180:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830414.856183:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3850 00000100:00080000:2.0:1787830414.856191:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830414.856195:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830414.856198:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830414.856208:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cf00 x1874672086049920/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830414.856217:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830414.856219:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787830414.856221:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830414.856225:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3850 00000100:00080000:2.0:1787830414.856228:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830414.856231:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830414.856235:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:2.0:1787830414.856246:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f0c0 x1874672086050048/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830414.856254:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830414.856258:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787830414.856259:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830414.856263:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3850 00000100:00080000:2.0:1787830414.856266:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830414.856269:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830414.856273:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:2.0:1787830414.856283:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f840 x1874672086050176/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830414.856291:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830414.856294:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787830414.856296:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830414.856300:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3850 00000100:00080000:2.0:1787830414.856303:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830414.856306:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830414.856310:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830414.856319:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c3c0 x1874672086050304/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830414.856327:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830414.856330:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787830414.856331:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830414.856335:0:479:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3850 00000100:00080000:2.0:1787830414.856338:0:479:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830414.856341:0:479:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830414.856345:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830414.856354:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c780 x1874672086050432/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830414.856362:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830414.856365:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787830414.856367:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830414.856370:0:479:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3850 00000100:00080000:2.0:1787830414.856374:0:479:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830414.856389:0:479:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830414.856394:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:2.0:1787830414.856404:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f480 x1874672086050560/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830414.856413:0:479:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830414.856415:0:479:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787830414.856417:0:479:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830414.856420:0:479:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3850 00000100:00080000:2.0:1787830414.856423:0:479:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830414.856427:0:479:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830414.856431:0:479:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:2.0:1787830414.856440:0:479:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498fc00 x1874672086050688/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787830414.856835:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672086049664:192.168.203.141@tcp:250:kworker.0 00000800:00000100:1.0:1787830414.856903:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830414.856941:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830414/real 1787830414] req@ffff9e35c498e580 x1874672086049664/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830439 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830414.856958:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e580 x1874672086049664/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830439 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830414.856967:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830414.856971:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830414.856975:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672086049664:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1787830414.856982:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086049792:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787830414.857003:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830414.857015:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830414/real 1787830414] req@ffff9e35c498e1c0 x1874672086049792/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830439 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830414.857026:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e1c0 x1874672086049792/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830439 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830414.857034:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830414.857038:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830414.857041:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086049792:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787830414.857047:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086049920:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787830414.857066:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830414.857077:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830414/real 1787830414] req@ffff9e35c498cf00 x1874672086049920/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830439 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830414.857087:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498cf00 x1874672086049920/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830439 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830414.857094:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830414.857097:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830414.857099:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086049920:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787830414.857104:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086050048:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787830414.857120:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830414.857130:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830414/real 1787830414] req@ffff9e35c498f0c0 x1874672086050048/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830439 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830414.857141:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f0c0 x1874672086050048/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830439 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830414.857148:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830414.857151:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830414.857154:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086050048:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830414.857160:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086050176:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787830414.857178:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830414.857188:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830414/real 1787830414] req@ffff9e35c498f840 x1874672086050176/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830439 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830414.857198:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f840 x1874672086050176/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830439 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830414.857212:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830414.857216:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830414.857219:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086050176:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830414.857224:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086050304:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787830414.857247:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830414.857257:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830414/real 1787830414] req@ffff9e35c498c3c0 x1874672086050304/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830439 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830414.857267:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c3c0 x1874672086050304/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830439 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830414.857274:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830414.857277:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830414.857280:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086050304:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787830414.857286:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086050432:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787830414.857304:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830414.857314:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830414/real 1787830414] req@ffff9e35c498c780 x1874672086050432/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830439 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830414.857327:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c780 x1874672086050432/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830439 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830414.857336:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830414.857339:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830414.857343:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086050432:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787830414.857349:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086050560:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787830414.857365:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830414.857376:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830414/real 1787830414] req@ffff9e35c498f480 x1874672086050560/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830439 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830414.857399:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f480 x1874672086050560/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830439 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830414.857408:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830414.857411:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830414.857414:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086050560:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830414.857419:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086050688:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787830414.857449:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830414.857460:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830414/real 1787830414] req@ffff9e35c498fc00 x1874672086050688/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830439 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830414.857472:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498fc00 x1874672086050688/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830439 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830414.857479:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830414.857482:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830414.857485:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086050688:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830414.983963:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:2.0:1787830415.048222:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787830415.048238:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787830415.048440:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787830416.007937:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830416.071927:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787830416.071927:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787830416.072262:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830417.031935:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830417.095931:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787830417.095945:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:0.0:1787830417.095969:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830418.055937:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830418.119957:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787830418.121029:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787830418.121086:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787830419.079952:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:3.0:1787830419.143985:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787830419.144100:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787830419.144153:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787830419.975984:0:27971:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830419.976219:0:27971:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787830419.976224:0:27971:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830419.976235:0:27971:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3855 00000100:00080000:2.0:1787830419.976247:0:27971:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830419.976254:0:27971:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787830419.976292:0:27971:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498ed00 x1874672086050816/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830419.976320:0:27971:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830419.976324:0:27971:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787830419.976326:0:27971:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830419.976330:0:27971:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3855 00000100:00080000:2.0:1787830419.976334:0:27971:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830419.976338:0:27971:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830419.976359:0:27971:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830419.976373:0:27971:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c000 x1874672086050944/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787830419.976381:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672086050816:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1787830419.976387:0:27971:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830419.976391:0:27971:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787830419.976392:0:27971:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830419.976397:0:27971:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3855 00000100:00080000:2.0:1787830419.976400:0:27971:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830419.976404:0:27971:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830419.976409:0:27971:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830419.976420:0:27971:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e940 x1874672086051072/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830419.976429:0:27971:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830419.976432:0:27971:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787830419.976433:0:27971:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830419.976436:0:27971:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3855 00000100:00080000:2.0:1787830419.976440:0:27971:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830419.976443:0:27971:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830419.976448:0:27971:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:2.0:1787830419.976462:0:27971:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498d680 x1874672086051200/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830419.976478:0:27971:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830419.976481:0:27971:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787830419.976483:0:27971:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830419.976487:0:27971:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3855 00000100:00080000:2.0:1787830419.976491:0:27971:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830419.976495:0:27971:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830419.976500:0:27971:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:2.0:1787830419.976513:0:27971:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cb40 x1874672086051328/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787830419.976514:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1787830419.976525:0:27971:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830419.976529:0:27971:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787830419.976530:0:27971:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830419.976535:0:27971:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3855 00000100:00080000:2.0:1787830419.976538:0:27971:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830419.976542:0:27971:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830419.976546:0:27971:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830419.976556:0:27971:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498de00 x1874672086051456/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787830419.976564:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830419/real 1787830419] req@ffff9e35c498ed00 x1874672086050816/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830444 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830419.976565:0:27971:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830419.976573:0:27971:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787830419.976576:0:27971:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830419.976584:0:27971:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3855 00000100:00080000:2.0:1787830419.976589:0:27971:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830419.976593:0:27971:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830419.976598:0:27971:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830419.976599:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498ed00 x1874672086050816/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830444 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830419.976613:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830419.976617:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830419.976624:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672086050816:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1787830419.976624:0:27971:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498fc00 x1874672086051584/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830419.976640:0:27971:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830419.976644:0:27971:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787830419.976646:0:27971:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830419.976650:0:27971:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3855 00000100:00080000:2.0:1787830419.976654:0:27971:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830419.976657:0:27971:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830419.976661:0:27971:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:1.0:1787830419.976669:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086050944:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787830419.976674:0:27971:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f480 x1874672086051712/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830419.976686:0:27971:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830419.976689:0:27971:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787830419.976691:0:27971:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830419.976695:0:27971:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3855 00000100:00080000:2.0:1787830419.976698:0:27971:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830419.976702:0:27971:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830419.976706:0:27971:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:2.0:1787830419.976716:0:27971:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c780 x1874672086051840/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787830419.976940:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830419.976970:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830419/real 1787830419] req@ffff9e35c498c000 x1874672086050944/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830444 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830419.976983:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c000 x1874672086050944/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830444 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830419.976991:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830419.976995:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830419.977147:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086050944:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787830419.977158:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086051072:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787830419.977184:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830419.977196:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830419/real 1787830419] req@ffff9e35c498e940 x1874672086051072/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830444 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830419.977218:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e940 x1874672086051072/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830444 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830419.977230:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830419.977234:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830419.977237:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086051072:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787830419.977243:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086051200:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787830419.977264:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830419.977276:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830419/real 1787830419] req@ffff9e35c498d680 x1874672086051200/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830444 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830419.977288:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498d680 x1874672086051200/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830444 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830419.977296:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830419.977299:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830419.977301:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086051200:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830419.977307:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086051328:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787830419.977326:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830419.977337:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830419/real 1787830419] req@ffff9e35c498cb40 x1874672086051328/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830444 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830419.977361:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498cb40 x1874672086051328/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830444 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830419.977370:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830419.977373:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830419.977376:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086051328:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830419.977381:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086051456:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787830419.977412:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830419.977424:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830419/real 1787830419] req@ffff9e35c498de00 x1874672086051456/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830444 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830419.977435:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498de00 x1874672086051456/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830444 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830419.977442:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830419.977445:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830419.977448:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086051456:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787830419.977454:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086051584:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787830419.977474:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830419.977485:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830419/real 1787830419] req@ffff9e35c498fc00 x1874672086051584/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830444 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830419.977495:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498fc00 x1874672086051584/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830444 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830419.977509:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830419.977513:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830419.977516:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086051584:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787830419.977541:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086051712:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787830419.977561:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830419.977572:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830419/real 1787830419] req@ffff9e35c498f480 x1874672086051712/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830444 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830419.977583:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f480 x1874672086051712/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830444 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830419.977592:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830419.977595:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830419.977598:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086051712:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830419.977604:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086051840:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787830419.977622:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830419.977633:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830419/real 1787830419] req@ffff9e35c498c780 x1874672086051840/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830444 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830419.977643:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c780 x1874672086051840/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830444 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830419.977661:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830419.977665:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830419.977669:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086051840:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830420.104152:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:0.0:1787830420.168038:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830420.168047:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787830420.168391:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787830421.128079:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830421.192019:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787830421.192027:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787830421.192035:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830422.153015:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787830422.215869:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787830422.216637:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830422.216898:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787830423.176096:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787830423.239933:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787830423.240199:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787830423.240936:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830424.200465:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787830424.263970:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830424.264278:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787830424.264384:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00080000:2.0:1787830425.095980:0:27971:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830425.095995:0:27971:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787830425.095999:0:27971:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830425.096011:0:27971:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3860 00000100:00080000:2.0:1787830425.096020:0:27971:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830425.096027:0:27971:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787830425.096062:0:27971:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c3c0 x1874672086051968/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830425.096096:0:27971:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830425.096100:0:27971:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787830425.096102:0:27971:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830425.096106:0:27971:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3860 00000100:00080000:2.0:1787830425.096110:0:27971:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830425.096114:0:27971:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830425.096125:0:27971:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830425.096139:0:27971:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f840 x1874672086052096/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830425.096148:0:27971:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830425.096152:0:27971:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787830425.096153:0:27971:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830425.096157:0:27971:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3860 00000100:00080000:2.0:1787830425.096160:0:27971:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830425.096163:0:27971:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830425.096167:0:27971:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830425.096177:0:27971:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f0c0 x1874672086052224/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830425.096184:0:27971:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830425.096187:0:27971:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787830425.096189:0:27971:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830425.096192:0:27971:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3860 00000100:00080000:2.0:1787830425.096195:0:27971:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830425.096199:0:27971:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830425.096203:0:27971:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:1.0:1787830425.096207:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672086051968:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1787830425.096216:0:27971:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cf00 x1874672086052352/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830425.096227:0:27971:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830425.096230:0:27971:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787830425.096232:0:27971:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830425.096235:0:27971:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3860 00000100:00080000:2.0:1787830425.096239:0:27971:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830425.096242:0:27971:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830425.096247:0:27971:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:2.0:1787830425.096258:0:27971:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e1c0 x1874672086052480/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830425.096268:0:27971:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1787830425.096273:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1787830425.096274:0:27971:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787830425.096277:0:27971:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830425.096283:0:27971:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3860 00000100:00080000:2.0:1787830425.096287:0:27971:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830425.096291:0:27971:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830425.096295:0:27971:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830425.096318:0:27971:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e580 x1874672086052608/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830425.096329:0:27971:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787830425.096331:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830425/real 1787830425] req@ffff9e35c498c3c0 x1874672086051968/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830450 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830425.096332:0:27971:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787830425.096336:0:27971:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830425.096343:0:27971:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3860 00000100:00080000:2.0:1787830425.096349:0:27971:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830425.096353:0:27971:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787830425.096358:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c3c0 x1874672086051968/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830450 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1787830425.096359:0:27971:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00080000:1.0:1787830425.096373:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:2.0:1787830425.096376:0:27971:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c780 x1874672086052736/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830425.096380:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830425.096387:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672086051968:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1787830425.096396:0:27971:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787830425.096400:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086052096:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1787830425.096403:0:27971:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787830425.096407:0:27971:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830425.096413:0:27971:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3860 00000100:00080000:2.0:1787830425.096417:0:27971:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830425.096421:0:27971:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830425.096427:0:27971:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000800:00000100:1.0:1787830425.096428:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787830425.096443:0:27971:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f480 x1874672086052864/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787830425.096445:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830425/real 1787830425] req@ffff9e35c498f840 x1874672086052096/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830450 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830425.096473:0:27971:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787830425.096475:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f840 x1874672086052096/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830450 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830425.096478:0:27971:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787830425.096482:0:27971:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830425.096492:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830425.096492:0:27971:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3860 00000100:00080000:2.0:1787830425.096498:0:27971:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830425.096499:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830425.096505:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086052096:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1787830425.096509:0:27971:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787830425.096515:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086052224:192.168.203.141@tcp:38:kworker.0 00000008:00100000:2.0:1787830425.096516:0:27971:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:2.0:1787830425.096533:0:27971:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498fc00 x1874672086052992/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787830425.096541:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830425.096554:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830425/real 1787830425] req@ffff9e35c498f0c0 x1874672086052224/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830450 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830425.096567:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f0c0 x1874672086052224/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830450 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830425.096588:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830425.096592:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830425.096595:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086052224:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787830425.096623:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086052352:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787830425.096642:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830425.096652:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830425/real 1787830425] req@ffff9e35c498cf00 x1874672086052352/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830450 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830425.096662:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498cf00 x1874672086052352/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830450 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830425.096671:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830425.096673:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830425.096676:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086052352:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830425.096681:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086052480:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787830425.096698:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830425.096708:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830425/real 1787830425] req@ffff9e35c498e1c0 x1874672086052480/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830450 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830425.096718:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e1c0 x1874672086052480/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830450 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830425.096725:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830425.096728:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830425.096730:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086052480:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830425.096735:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086052608:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787830425.096752:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830425.096762:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830425/real 1787830425] req@ffff9e35c498e580 x1874672086052608/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830450 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830425.096773:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e580 x1874672086052608/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830450 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830425.096781:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830425.096784:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830425.096787:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086052608:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787830425.096792:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086052736:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787830425.096852:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830425.096864:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830425/real 1787830425] req@ffff9e35c498c780 x1874672086052736/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830450 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830425.096882:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c780 x1874672086052736/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830450 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830425.096890:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830425.096893:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830425.096896:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086052736:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787830425.096901:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086052864:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787830425.096931:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830425.096942:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830425/real 1787830425] req@ffff9e35c498f480 x1874672086052864/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830450 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830425.096952:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f480 x1874672086052864/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830450 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830425.096959:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830425.096962:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830425.096964:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086052864:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830425.096970:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086052992:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787830425.096986:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830425.096995:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830425/real 1787830425] req@ffff9e35c498fc00 x1874672086052992/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830450 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830425.097005:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498fc00 x1874672086052992/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830450 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830425.097017:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830425.097020:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830425.097023:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086052992:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830425.224172:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787830425.287934:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787830425.287964:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787830425.287981:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830426.247924:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830426.312067:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787830426.312067:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787830426.312127:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787830427.271978:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787830427.335932:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787830427.335933:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787830427.335978:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830428.295937:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830428.359915:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787830428.359915:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787830428.359960:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830429.319934:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787830429.383935:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787830429.384040:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830429.384270:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00080000:1.0:1787830430.215994:0:27971:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830430.216003:0:27971:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787830430.216008:0:27971:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830430.216019:0:27971:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3865 00000100:00080000:1.0:1787830430.216029:0:27971:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830430.216036:0:27971:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787830430.216130:0:27971:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6940 x1874672086053120/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830430.216162:0:27971:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830430.216165:0:27971:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787830430.216167:0:27971:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830430.216171:0:27971:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3865 00000100:00080000:1.0:1787830430.216176:0:27971:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830430.216181:0:27971:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830430.216191:0:27971:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830430.216206:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672086053120:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1787830430.216217:0:27971:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f70c0 x1874672086053248/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830430.216229:0:27971:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830430.216233:0:27971:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787830430.216235:0:27971:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1787830430.216280:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830430.216333:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830430/real 1787830430] req@ffff9e35da6f6940 x1874672086053120/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830455 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830430.216351:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6940 x1874672086053120/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830455 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830430.216362:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830430.216367:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830430.216371:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672086053120:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1787830430.216398:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086053248:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830430.216424:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830430.216436:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830430/real 1787830430] req@ffff9e35da6f70c0 x1874672086053248/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830455 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830430.216448:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f70c0 x1874672086053248/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830455 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830430.216456:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830430.216459:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830430.216462:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086053248:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787830430.216544:0:27971:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3865 00000100:00080000:1.0:1787830430.216549:0:27971:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830430.216553:0:27971:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830430.216559:0:27971:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830430.216572:0:27971:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4b40 x1874672086053376/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830430.216593:0:27971:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830430.216597:0:27971:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787830430.216599:0:27971:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830430.216603:0:27971:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3865 00000100:00080000:1.0:1787830430.216607:0:27971:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830430.216611:0:27971:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830430.216617:0:27971:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:1.0:1787830430.216627:0:27971:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5e00 x1874672086053504/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830430.216635:0:27971:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830430.216638:0:27971:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787830430.216640:0:27971:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830430.216644:0:27971:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3865 00000100:00080000:1.0:1787830430.216647:0:27971:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830430.216650:0:27971:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830430.216655:0:27971:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:1.0:1787830430.216666:0:27971:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5a40 x1874672086053632/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830430.216685:0:27971:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830430.216688:0:27971:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787830430.216690:0:27971:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830430.216693:0:27971:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3865 00000100:00080000:1.0:1787830430.216696:0:27971:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830430.216700:0:27971:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830430.216704:0:27971:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830430.216715:0:27971:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f70c0 x1874672086053760/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830430.216725:0:27971:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830430.216729:0:27971:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787830430.216731:0:27971:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830430.216735:0:27971:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3865 00000100:00080000:1.0:1787830430.216739:0:27971:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830430.216743:0:27971:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830430.216747:0:27971:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830430.216761:0:27971:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6940 x1874672086053888/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830430.216772:0:27971:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787830430.216773:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086053376:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787830430.216779:0:27971:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787830430.216782:0:27971:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830430.216787:0:27971:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3865 00000100:00080000:1.0:1787830430.216790:0:27971:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830430.216795:0:27971:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:0.0:1787830430.216826:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830430.216842:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830430/real 1787830430] req@ffff9e35da6f4b40 x1874672086053376/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830455 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830430.216854:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4b40 x1874672086053376/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830455 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830430.216862:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830430.216865:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830430.216867:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086053376:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830430.216873:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086053504:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830430.216893:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:1.0:1787830430.216901:0:27971:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:0.0:1787830430.216908:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830430/real 1787830430] req@ffff9e35da6f5e00 x1874672086053504/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830455 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830430.216917:0:27971:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4000 x1874672086054016/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830430.216923:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5e00 x1874672086053504/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830455 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830430.216942:0:27971:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830430.216943:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830430.216949:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:1.0:1787830430.216950:0:27971:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787830430.216954:0:27971:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787830430.216955:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086053504:192.168.203.141@tcp:8:kworker.0 00000100:00080000:1.0:1787830430.216962:0:27971:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3865 00000100:00100000:0.0:1787830430.216966:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086053632:192.168.203.141@tcp:8:kworker.0 00000100:00080000:1.0:1787830430.216970:0:27971:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830430.216977:0:27971:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830430.216982:0:27971:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000800:00000100:0.0:1787830430.216991:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830430.216997:0:27971:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4f00 x1874672086054144/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830430.217005:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830430/real 1787830430] req@ffff9e35da6f5a40 x1874672086053632/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830455 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830430.217026:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5a40 x1874672086053632/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830455 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830430.217034:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830430.217036:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830430.217039:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086053632:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830430.217044:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086053760:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830430.217060:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830430.217071:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830430/real 1787830430] req@ffff9e35da6f70c0 x1874672086053760/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830455 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830430.217087:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f70c0 x1874672086053760/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830455 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830430.217094:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830430.217097:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830430.217100:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086053760:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830430.217119:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086053888:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830430.217139:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830430.217149:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830430/real 1787830430] req@ffff9e35da6f6940 x1874672086053888/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830455 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830430.217158:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6940 x1874672086053888/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830455 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830430.217166:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830430.217169:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830430.217172:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086053888:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830430.217177:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086054016:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830430.217192:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830430.217201:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830430/real 1787830430] req@ffff9e35da6f4000 x1874672086054016/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830455 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830430.217211:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4000 x1874672086054016/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830455 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830430.217224:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830430.217226:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830430.217229:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086054016:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830430.217235:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086054144:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830430.217249:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830430.217258:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830430/real 1787830430] req@ffff9e35da6f4f00 x1874672086054144/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830455 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830430.217269:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4f00 x1874672086054144/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830455 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830430.217290:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830430.217293:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830430.217296:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086054144:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830430.344607:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787830430.407935:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787830430.408441:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830430.408521:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787830431.367934:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830431.431945:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787830431.432287:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830431.432295:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787830432.392113:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:1.0:1787830432.456142:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830432.456376:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787830432.456482:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787830433.415935:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787830433.479920:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787830433.479959:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787830433.479990:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787830434.439943:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:3.0:1787830434.504017:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787830434.504258:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830434.504366:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00080000:1.0:1787830435.335952:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830435.335964:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787830435.335967:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830435.335978:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3870 00000100:00080000:1.0:1787830435.335988:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830435.336004:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787830435.336036:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6580 x1874672086054272/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830435.336072:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830435.336075:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787830435.336077:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830435.336081:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3870 00000100:00080000:1.0:1787830435.336085:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830435.336089:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830435.336099:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830435.336115:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672086054272:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1787830435.336115:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6d00 x1874672086054400/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830435.336130:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830435.336143:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787830435.336145:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830435.336150:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3870 00000100:00080000:1.0:1787830435.336168:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830435.336171:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830435.336176:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1787830435.336179:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830435.336193:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7840 x1874672086054528/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830435.336203:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830435.336206:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787830435.336208:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830435.336213:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3870 00000100:00080000:1.0:1787830435.336217:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830435.336223:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787830435.336224:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830435/real 1787830435] req@ffff9e35da6f6580 x1874672086054272/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830460 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1787830435.336230:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:1.0:1787830435.336255:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f43c0 x1874672086054656/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830435.336270:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830435.336275:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:0.0:1787830435.336276:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6580 x1874672086054272/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830460 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830435.336278:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830435.336287:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3870 00000100:00080000:0.0:1787830435.336292:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830435.336293:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830435.336300:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:1.0:1787830435.336302:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787830435.336308:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672086054272:192.168.203.141@tcp:250:kworker.0 00000008:00100000:1.0:1787830435.336310:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:1.0:1787830435.336325:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f61c0 x1874672086054784/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830435.336335:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830435.336338:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787830435.336340:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830435.336345:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3870 00000100:00100000:0.0:1787830435.336349:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086054400:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787830435.336350:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830435.336358:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830435.336364:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830435.336378:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4780 x1874672086054912/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787830435.336382:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830435.336397:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830435/real 1787830435] req@ffff9e35da6f6d00 x1874672086054400/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830460 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830435.336406:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830435.336414:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787830435.336417:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787830435.336421:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6d00 x1874672086054400/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830460 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830435.336423:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3870 00000100:00080000:1.0:1787830435.336429:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830435.336436:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:0.0:1787830435.336437:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000008:00100000:1.0:1787830435.336442:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1787830435.336443:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830435.336450:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086054400:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787830435.336456:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7480 x1874672086055040/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830435.336460:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086054528:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787830435.336471:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830435.336473:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787830435.336475:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830435.336478:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3870 00000100:00080000:1.0:1787830435.336484:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1787830435.336489:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787830435.336489:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830435.336493:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:1.0:1787830435.336502:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6580 x1874672086055168/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830435.336506:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830435/real 1787830435] req@ffff9e35da6f7840 x1874672086054528/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830460 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830435.336510:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830435.336516:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787830435.336518:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830435.336524:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3870 00000100:00080000:1.0:1787830435.336527:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830435.336529:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787830435.336534:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7840 x1874672086054528/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830460 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1787830435.336534:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:1.0:1787830435.336551:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4f00 x1874672086055296/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830435.336554:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830435.336561:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830435.336567:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086054528:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830435.336576:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086054656:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830435.336596:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830435.336608:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830435/real 1787830435] req@ffff9e35da6f43c0 x1874672086054656/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830460 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830435.336619:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f43c0 x1874672086054656/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830460 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830435.336627:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830435.336630:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830435.336633:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086054656:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830435.336650:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086054784:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830435.336668:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830435.336679:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830435/real 1787830435] req@ffff9e35da6f61c0 x1874672086054784/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830460 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830435.336690:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f61c0 x1874672086054784/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830460 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830435.336699:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830435.336702:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830435.336705:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086054784:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830435.336710:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086054912:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830435.336727:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830435.336737:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830435/real 1787830435] req@ffff9e35da6f4780 x1874672086054912/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830460 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830435.336749:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4780 x1874672086054912/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830460 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830435.336757:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830435.336760:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830435.336763:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086054912:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830435.336768:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086055040:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830435.336785:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830435.336795:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830435/real 1787830435] req@ffff9e35da6f7480 x1874672086055040/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830460 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830435.336844:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7480 x1874672086055040/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830460 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830435.336852:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830435.336855:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830435.336858:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086055040:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830435.336864:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086055168:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830435.336882:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830435.336905:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830435/real 1787830435] req@ffff9e35da6f6580 x1874672086055168/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830460 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830435.336918:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6580 x1874672086055168/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830460 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830435.336927:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830435.336930:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830435.336933:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086055168:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830435.336939:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086055296:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830435.336956:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830435.336966:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830435/real 1787830435] req@ffff9e35da6f4f00 x1874672086055296/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830460 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830435.336977:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4f00 x1874672086055296/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830460 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830435.336985:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830435.336988:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830435.336991:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086055296:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830435.463940:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830435.527965:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787830435.527990:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830435.528109:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787830436.487955:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830436.551932:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787830436.551934:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787830436.552025:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830437.512038:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830437.575956:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:3.0:1787830437.576212:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787830437.576256:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830438.536086:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:3.0:1787830438.600029:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787830438.600110:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787830438.600139:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830439.559940:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830439.623966:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787830439.624039:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787830439.624240:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787830440.455987:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830440.455996:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787830440.456001:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830440.456015:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3875 00000100:00080000:2.0:1787830440.456025:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830440.456032:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787830440.456068:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498de00 x1874672086055424/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830440.456096:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830440.456099:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787830440.456101:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830440.456105:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3875 00000100:00080000:2.0:1787830440.456109:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830440.456113:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830440.456122:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830440.456134:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cb40 x1874672086055552/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830440.456144:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830440.456147:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787830440.456149:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830440.456153:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3875 00000100:00080000:2.0:1787830440.456155:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830440.456158:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830440.456161:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830440.456168:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498d680 x1874672086055680/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830440.456174:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830440.456176:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787830440.456177:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830440.456179:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3875 00000100:00080000:2.0:1787830440.456181:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830440.456184:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830440.456188:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:2.0:1787830440.456198:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e940 x1874672086055808/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830440.456219:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830440.456223:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787830440.456226:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787830440.456228:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672086055424:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1787830440.456232:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3875 00000100:00080000:2.0:1787830440.456239:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830440.456242:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830440.456246:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:2.0:1787830440.456259:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c000 x1874672086055936/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830440.456279:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830440.456282:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787830440.456284:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830440.456290:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3875 00000800:00000100:0.0:1787830440.456291:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1787830440.456296:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830440.456302:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830440.456308:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830440.456320:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498ed00 x1874672086056064/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830440.456330:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830440.456333:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787830440.456336:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787830440.456338:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830440/real 1787830440] req@ffff9e35c498de00 x1874672086055424/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830465 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830440.456342:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3875 00000100:00100000:0.0:1787830440.456361:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498de00 x1874672086055424/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830465 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830440.456361:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830440.456369:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830440.456377:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1787830440.456381:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830440.456388:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787830440.456392:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498fc00 x1874672086056192/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830440.456396:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672086055424:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1787830440.456404:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830440.456407:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:0.0:1787830440.456410:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086055552:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1787830440.456410:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830440.456417:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3875 00000100:00080000:2.0:1787830440.456419:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830440.456422:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830440.456425:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:2.0:1787830440.456434:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f480 x1874672086056320/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830440.456440:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1787830440.456442:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1787830440.456443:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787830440.456446:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830440.456449:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3875 00000100:00080000:2.0:1787830440.456452:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830440.456455:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830440.456458:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:0.0:1787830440.456463:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830440/real 1787830440] req@ffff9e35c498cb40 x1874672086055552/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830465 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830440.456467:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c780 x1874672086056448/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830440.456491:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498cb40 x1874672086055552/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830465 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830440.456503:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830440.456508:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830440.456513:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086055552:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830440.456520:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086055680:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830440.456541:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830440.456553:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830440/real 1787830440] req@ffff9e35c498d680 x1874672086055680/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830465 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830440.456565:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498d680 x1874672086055680/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830465 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830440.456579:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830440.456582:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830440.456586:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086055680:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830440.456612:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086055808:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830440.456631:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830440.456641:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830440/real 1787830440] req@ffff9e35c498e940 x1874672086055808/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830465 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830440.456653:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e940 x1874672086055808/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830465 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830440.456661:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830440.456664:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830440.456667:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086055808:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830440.456673:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086055936:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830440.456691:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830440.456701:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830440/real 1787830440] req@ffff9e35c498c000 x1874672086055936/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830465 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830440.456712:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c000 x1874672086055936/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830465 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830440.456719:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830440.456722:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830440.456726:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086055936:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830440.456731:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086056064:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830440.456749:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830440.456758:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830440/real 1787830440] req@ffff9e35c498ed00 x1874672086056064/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830465 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830440.456770:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498ed00 x1874672086056064/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830465 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830440.456778:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830440.456781:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830440.456784:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086056064:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830440.456790:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086056192:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830440.456860:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830440.456890:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830440/real 1787830440] req@ffff9e35c498fc00 x1874672086056192/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830465 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830440.456903:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498fc00 x1874672086056192/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830465 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830440.456912:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830440.456915:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830440.456918:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086056192:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830440.456924:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086056320:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830440.456943:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830440.456955:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830440/real 1787830440] req@ffff9e35c498f480 x1874672086056320/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830465 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830440.456967:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f480 x1874672086056320/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830465 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830440.456975:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830440.456978:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830440.456981:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086056320:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830440.456987:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086056448:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830440.457004:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830440.457014:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830440/real 1787830440] req@ffff9e35c498c780 x1874672086056448/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830465 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830440.457026:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c780 x1874672086056448/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830465 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830440.457034:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830440.457037:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830440.457040:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086056448:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830440.583947:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830440.647935:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787830440.648344:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830440.648385:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787830441.608067:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830441.672271:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:3.0:1787830441.672558:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787830441.673235:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830442.631912:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830442.695977:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787830442.696648:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787830442.696688:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830443.655947:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830443.719964:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787830443.720216:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830443.720219:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787830444.680026:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830444.743973:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787830444.743990:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787830444.744376:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787830445.575970:0:27971:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830445.575983:0:27971:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787830445.575986:0:27971:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830445.575998:0:27971:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3880 00000100:00080000:1.0:1787830445.576008:0:27971:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830445.576015:0:27971:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787830445.576049:0:27971:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4000 x1874672086056576/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830445.576074:0:27971:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830445.576078:0:27971:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787830445.576080:0:27971:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830445.576083:0:27971:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3880 00000100:00080000:1.0:1787830445.576086:0:27971:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830445.576090:0:27971:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830445.576100:0:27971:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830445.576112:0:27971:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6940 x1874672086056704/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830445.576118:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672086056576:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787830445.576122:0:27971:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830445.576126:0:27971:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787830445.576127:0:27971:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830445.576131:0:27971:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3880 00000100:00080000:1.0:1787830445.576133:0:27971:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830445.576136:0:27971:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830445.576139:0:27971:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830445.576148:0:27971:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f70c0 x1874672086056832/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830445.576154:0:27971:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830445.576156:0:27971:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787830445.576157:0:27971:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830445.576159:0:27971:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3880 00000100:00080000:1.0:1787830445.576179:0:27971:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830445.576182:0:27971:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830445.576185:0:27971:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000800:00000100:0.0:1787830445.576190:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830445.576205:0:27971:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5a40 x1874672086056960/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830445.576215:0:27971:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830445.576218:0:27971:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787830445.576220:0:27971:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830445.576224:0:27971:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3880 00000100:00080000:1.0:1787830445.576228:0:27971:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830445.576232:0:27971:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787830445.576237:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830445/real 1787830445] req@ffff9e35da6f4000 x1874672086056576/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830470 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1787830445.576239:0:27971:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:1.0:1787830445.576256:0:27971:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5e00 x1874672086057088/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830445.576266:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4000 x1874672086056576/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830470 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830445.576267:0:27971:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830445.576275:0:27971:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787830445.576278:0:27971:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830445.576286:0:27971:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3880 00000100:00080000:0.0:1787830445.576287:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830445.576289:0:27971:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830445.576294:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:1.0:1787830445.576297:0:27971:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787830445.576302:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672086056576:192.168.203.141@tcp:250:kworker.0 00000008:00100000:1.0:1787830445.576304:0:27971:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830445.576318:0:27971:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4b40 x1874672086057216/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830445.576326:0:27971:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830445.576329:0:27971:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787830445.576332:0:27971:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787830445.576333:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086056704:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787830445.576337:0:27971:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3880 00000100:00080000:1.0:1787830445.576343:0:27971:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830445.576347:0:27971:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830445.576351:0:27971:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830445.576362:0:27971:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4000 x1874672086057344/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787830445.576370:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787830445.576377:0:27971:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830445.576382:0:27971:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787830445.576384:0:27971:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787830445.576388:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830445/real 1787830445] req@ffff9e35da6f6940 x1874672086056704/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830470 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830445.576391:0:27971:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3880 00000100:00080000:1.0:1787830445.576397:0:27971:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830445.576403:0:27971:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787830445.576408:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6940 x1874672086056704/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830470 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1787830445.576408:0:27971:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:1.0:1787830445.576422:0:27971:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4f00 x1874672086057472/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830445.576430:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830445.576435:0:27971:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830445.576438:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:1.0:1787830445.576441:0:27971:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:0.0:1787830445.576446:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086056704:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787830445.576446:0:27971:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830445.576454:0:27971:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3880 00000100:00100000:0.0:1787830445.576458:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086056832:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787830445.576460:0:27971:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830445.576468:0:27971:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830445.576473:0:27971:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000800:00000100:0.0:1787830445.576485:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830445.576486:0:27971:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6580 x1874672086057600/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830445.576501:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830445/real 1787830445] req@ffff9e35da6f70c0 x1874672086056832/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830470 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830445.576514:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f70c0 x1874672086056832/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830470 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830445.576527:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830445.576530:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830445.576533:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086056832:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830445.576538:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086056960:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830445.576556:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830445.576567:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830445/real 1787830445] req@ffff9e35da6f5a40 x1874672086056960/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830470 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830445.576577:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5a40 x1874672086056960/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830470 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830445.576584:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830445.576587:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830445.576590:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086056960:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830445.576596:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086057088:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830445.576612:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830445.576622:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830445/real 1787830445] req@ffff9e35da6f5e00 x1874672086057088/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830470 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830445.576632:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5e00 x1874672086057088/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830470 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830445.576639:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830445.576642:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830445.576645:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086057088:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830445.576651:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086057216:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830445.576668:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830445.576677:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830445/real 1787830445] req@ffff9e35da6f4b40 x1874672086057216/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830470 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830445.576687:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4b40 x1874672086057216/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830470 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830445.576695:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830445.576697:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830445.576701:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086057216:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830445.576721:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086057344:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830445.576748:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830445.576759:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830445/real 1787830445] req@ffff9e35da6f4000 x1874672086057344/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830470 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830445.576771:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4000 x1874672086057344/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830470 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830445.576778:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830445.576781:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830445.576784:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086057344:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830445.576789:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086057472:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830445.576892:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830445.576907:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830445/real 1787830445] req@ffff9e35da6f4f00 x1874672086057472/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830470 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830445.576919:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4f00 x1874672086057472/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830470 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830445.576928:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830445.576931:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830445.576934:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086057472:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830445.576939:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086057600:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830445.576952:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830445.576960:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830445/real 1787830445] req@ffff9e35da6f6580 x1874672086057600/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830470 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830445.576969:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6580 x1874672086057600/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830470 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830445.576977:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830445.576980:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830445.576983:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086057600:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830445.703936:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830445.767935:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:3.0:1787830445.767935:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787830445.768004:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830446.727950:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830446.791946:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787830446.792991:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830446.793135:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787830447.751961:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830447.815930:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:3.0:1787830447.816004:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787830447.816486:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830448.775927:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787830448.839949:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830448.840026:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787830448.840037:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787830449.799953:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830449.863914:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787830449.863931:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787830449.863953:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00080000:2.0:1787830450.696005:0:27971:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830450.696018:0:27971:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787830450.696022:0:27971:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830450.696033:0:27971:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3886 00000100:00080000:2.0:1787830450.696043:0:27971:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830450.696049:0:27971:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787830450.696082:0:27971:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e580 x1874672086057728/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830450.696123:0:27971:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830450.696144:0:27971:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787830450.696147:0:27971:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830450.696153:0:27971:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3886 00000100:00080000:2.0:1787830450.696157:0:27971:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830450.696161:0:27971:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830450.696171:0:27971:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830450.696178:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672086057728:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1787830450.696187:0:27971:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e1c0 x1874672086057856/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830450.696198:0:27971:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830450.696201:0:27971:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787830450.696202:0:27971:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830450.696205:0:27971:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3886 00000100:00080000:2.0:1787830450.696209:0:27971:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830450.696212:0:27971:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830450.696217:0:27971:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830450.696226:0:27971:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cf00 x1874672086057984/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830450.696235:0:27971:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830450.696238:0:27971:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787830450.696239:0:27971:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830450.696243:0:27971:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3886 00000100:00080000:2.0:1787830450.696246:0:27971:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830450.696250:0:27971:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830450.696255:0:27971:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:2.0:1787830450.696265:0:27971:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f0c0 x1874672086058112/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830450.696271:0:27971:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830450.696273:0:27971:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787830450.696274:0:27971:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830450.696277:0:27971:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3886 00000100:00080000:2.0:1787830450.696279:0:27971:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830450.696281:0:27971:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830450.696285:0:27971:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:0.0:1787830450.696289:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086057856:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787830450.696295:0:27971:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f840 x1874672086058240/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830450.696306:0:27971:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830450.696308:0:27971:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787830450.696309:0:27971:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830450.696312:0:27971:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3886 00000100:00080000:2.0:1787830450.696315:0:27971:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830450.696317:0:27971:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830450.696319:0:27971:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830450.696327:0:27971:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c3c0 x1874672086058368/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830450.696333:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086057984:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1787830450.696334:0:27971:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830450.696339:0:27971:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787830450.696343:0:27971:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830450.696346:0:27971:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3886 00000100:00080000:2.0:1787830450.696349:0:27971:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830450.696353:0:27971:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830450.696357:0:27971:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830450.696367:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086058112:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787830450.696372:0:27971:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c780 x1874672086058496/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830450.696384:0:27971:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830450.696387:0:27971:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787830450.696390:0:27971:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787830450.696394:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086058240:192.168.203.141@tcp:8:kworker.0 00000100:00080000:2.0:1787830450.696395:0:27971:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3886 00000100:00080000:2.0:1787830450.696403:0:27971:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830450.696409:0:27971:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830450.696414:0:27971:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:0.0:1787830450.696422:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086058368:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787830450.696429:0:27971:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f480 x1874672086058624/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830450.696440:0:27971:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830450.696443:0:27971:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787830450.696445:0:27971:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787830450.696450:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086058496:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1787830450.696451:0:27971:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3886 00000100:00080000:2.0:1787830450.696459:0:27971:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830450.696464:0:27971:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830450.696469:0:27971:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:0.0:1787830450.696482:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086058624:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787830450.696485:0:27971:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498fc00 x1874672086058752/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830450.696511:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086058752:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830450.823925:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830450.887932:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787830450.887947:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787830450.887976:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830451.847989:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830451.911946:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:3.0:1787830451.912481:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787830451.912528:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830452.871972:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787830452.936058:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787830452.936091:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830452.936181:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000400:00000100:3.0:1787830453.768484:0:2398:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.141:988 00000400:00000100:3.0:1787830453.768530:0:2398:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.141@tcp at host 192.168.203.141 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1787830453.768545:0:2398:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:0.0:1787830453.773399:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830450/real 1787830453] req@ffff9e35c498e580 x1874672086057728/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830475 ref 2 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830453.773434:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e580 x1874672086057728/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830475 ref 2 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830453.773449:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830453.773454:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830453.773460:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672086057728:192.168.203.141@tcp:250:kworker.0 00000800:00000100:3.0:1787830453.773474:0:2398:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000800:00000100:3.0:1787830453.773498:0:2398:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:0.0:1787830453.773505:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830450/real 1787830453] req@ffff9e35c498fc00 x1874672086058752/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830475 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:3.0:1787830453.773511:0:2398:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1787830453.773522:0:2398:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:0.0:1787830453.773530:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498fc00 x1874672086058752/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830475 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:3.0:1787830453.773535:0:2398:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00080000:0.0:1787830453.773547:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000800:00000100:3.0:1787830453.773548:0:2398:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00080000:0.0:1787830453.773558:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000800:00000100:3.0:1787830453.773563:0:2398:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:0.0:1787830453.773565:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086058752:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1787830453.773575:0:2398:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:0.0:1787830453.773577:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830450/real 1787830453] req@ffff9e35c498f840 x1874672086058240/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830475 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830453.773593:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f840 x1874672086058240/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830475 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830453.773602:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830453.773607:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830453.773610:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086058240:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830453.773616:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830450/real 1787830453] req@ffff9e35c498c3c0 x1874672086058368/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830475 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830453.773628:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c3c0 x1874672086058368/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830475 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830453.773636:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830453.773639:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830453.773643:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086058368:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830453.773649:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830450/real 1787830453] req@ffff9e35c498c780 x1874672086058496/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830475 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830453.773659:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c780 x1874672086058496/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830475 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830453.773667:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830453.773671:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830453.773673:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086058496:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830453.773680:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830450/real 1787830453] req@ffff9e35c498e1c0 x1874672086057856/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830475 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830453.773690:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e1c0 x1874672086057856/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830475 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830453.773698:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830453.773702:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830453.773705:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086057856:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830453.773711:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830450/real 1787830453] req@ffff9e35c498cf00 x1874672086057984/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830475 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830453.773722:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498cf00 x1874672086057984/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830475 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830453.773729:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830453.773732:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830453.773735:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086057984:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830453.773741:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830450/real 1787830453] req@ffff9e35c498f0c0 x1874672086058112/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830475 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830453.773752:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f0c0 x1874672086058112/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830475 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830453.773759:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830453.773763:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830453.773766:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086058112:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830453.773791:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830450/real 1787830453] req@ffff9e35c498f480 x1874672086058624/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830475 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830453.773821:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f480 x1874672086058624/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830475 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830453.773830:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830453.773839:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830453.773842:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086058624:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830453.895938:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:1.0:1787830453.959999:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830453.960050:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787830453.960825:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787830454.919946:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787830454.983957:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787830454.984146:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787830454.984659:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00080000:2.0:1787830455.816460:0:27971:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830455.816472:0:27971:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787830455.816476:0:27971:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830455.816488:0:27971:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3891 00000100:00080000:2.0:1787830455.816498:0:27971:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830455.816505:0:27971:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787830455.816546:0:27971:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498ed00 x1874672086058880/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830455.816581:0:27971:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830455.816585:0:27971:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787830455.816587:0:27971:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830455.816591:0:27971:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3891 00000100:00080000:2.0:1787830455.816595:0:27971:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830455.816599:0:27971:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830455.816609:0:27971:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830455.816622:0:27971:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c000 x1874672086059008/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830455.816632:0:27971:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830455.816635:0:27971:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787830455.816638:0:27971:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830455.816641:0:27971:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3891 00000100:00080000:2.0:1787830455.816645:0:27971:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830455.816648:0:27971:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830455.816652:0:27971:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830455.816662:0:27971:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e940 x1874672086059136/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830455.816671:0:27971:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830455.816674:0:27971:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787830455.816675:0:27971:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830455.816680:0:27971:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3891 00000100:00080000:2.0:1787830455.816682:0:27971:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830455.816686:0:27971:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830455.816690:0:27971:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:2.0:1787830455.816700:0:27971:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498d680 x1874672086059264/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830455.816708:0:27971:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830455.816711:0:27971:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787830455.816713:0:27971:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830455.816717:0:27971:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3891 00000100:00080000:2.0:1787830455.816727:0:27971:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830455.816731:0:27971:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830455.816735:0:27971:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:2.0:1787830455.816746:0:27971:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cb40 x1874672086059392/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830455.816755:0:27971:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830455.816758:0:27971:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787830455.816759:0:27971:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830455.816763:0:27971:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3891 00000100:00080000:2.0:1787830455.816766:0:27971:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830455.816770:0:27971:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830455.816774:0:27971:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830455.816784:0:27971:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498de00 x1874672086059520/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830455.816793:0:27971:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830455.816796:0:27971:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787830455.816838:0:27971:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830455.816842:0:27971:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3891 00000100:00080000:2.0:1787830455.816845:0:27971:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830455.816849:0:27971:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830455.816854:0:27971:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830455.816865:0:27971:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f480 x1874672086059648/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830455.816873:0:27971:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830455.816876:0:27971:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787830455.816878:0:27971:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830455.816882:0:27971:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3891 00000100:00080000:2.0:1787830455.816885:0:27971:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830455.816889:0:27971:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830455.816893:0:27971:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:2.0:1787830455.816903:0:27971:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f0c0 x1874672086059776/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830455.816911:0:27971:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830455.816915:0:27971:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787830455.816916:0:27971:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830455.816920:0:27971:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3891 00000100:00080000:2.0:1787830455.816923:0:27971:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830455.816926:0:27971:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830455.816930:0:27971:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:2.0:1787830455.816940:0:27971:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cf00 x1874672086059904/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830455.817000:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672086058880:192.168.203.141@tcp:250:kworker.0 00000800:00000100:0.0:1787830455.817061:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830455.817108:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830455/real 1787830455] req@ffff9e35c498ed00 x1874672086058880/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830480 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830455.817125:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498ed00 x1874672086058880/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830480 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830455.817136:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830455.817140:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830455.817144:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672086058880:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1787830455.817151:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086059008:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830455.817173:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830455.817185:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830455/real 1787830455] req@ffff9e35c498c000 x1874672086059008/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830480 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830455.817197:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c000 x1874672086059008/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830480 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830455.817206:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830455.817209:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830455.817212:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086059008:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830455.817218:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086059136:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830455.817237:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830455.817249:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830455/real 1787830455] req@ffff9e35c498e940 x1874672086059136/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830480 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830455.817260:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e940 x1874672086059136/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830480 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830455.817268:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830455.817271:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830455.817274:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086059136:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830455.817279:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086059264:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830455.817297:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830455.817308:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830455/real 1787830455] req@ffff9e35c498d680 x1874672086059264/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830480 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830455.817317:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498d680 x1874672086059264/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830480 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830455.817325:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830455.817328:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830455.817331:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086059264:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830455.817336:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086059392:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830455.817354:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830455.817364:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830455/real 1787830455] req@ffff9e35c498cb40 x1874672086059392/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830480 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830455.817375:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498cb40 x1874672086059392/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830480 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830455.817383:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830455.817385:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830455.817388:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086059392:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830455.817394:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086059520:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830455.817412:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830455.817423:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830455/real 1787830455] req@ffff9e35c498de00 x1874672086059520/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830480 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830455.817433:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498de00 x1874672086059520/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830480 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830455.817441:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830455.817444:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830455.817447:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086059520:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830455.817452:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086059648:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830455.817470:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830455.817481:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830455/real 1787830455] req@ffff9e35c498f480 x1874672086059648/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830480 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830455.817493:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f480 x1874672086059648/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830480 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830455.817501:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830455.817504:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830455.817506:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086059648:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830455.817512:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086059776:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830455.817529:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830455.817539:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830455/real 1787830455] req@ffff9e35c498f0c0 x1874672086059776/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830480 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830455.817549:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f0c0 x1874672086059776/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830480 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830455.817556:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830455.817559:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830455.817562:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086059776:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830455.817568:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086059904:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830455.817587:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830455.817597:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830455/real 1787830455] req@ffff9e35c498cf00 x1874672086059904/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830480 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830455.817608:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498cf00 x1874672086059904/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830480 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830455.817615:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830455.817618:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830455.817621:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086059904:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830455.944259:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787830456.008005:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787830456.008020:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830456.008263:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787830456.967938:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787830457.031948:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787830457.032037:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830457.032239:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787830457.991926:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830458.055959:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787830458.055962:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787830458.057852:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830459.016096:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830459.079924:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787830459.079924:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787830459.079996:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830460.043216:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830460.103928:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787830460.104206:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830460.104965:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00080000:2.0:1787830460.936112:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830460.936124:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787830460.936128:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830460.936138:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3896 00000100:00080000:2.0:1787830460.936148:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830460.936156:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787830460.936195:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e1c0 x1874672086060032/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830460.936266:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672086060032:192.168.203.141@tcp:250:kworker.0 00000800:00000100:0.0:1787830460.936328:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1787830460.936358:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830460.936361:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787830460.936363:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830460.936367:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3896 00000100:00080000:2.0:1787830460.936371:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830460.936375:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830460.936385:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830460.936398:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c780 x1874672086060160/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830460.936507:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830460/real 1787830460] req@ffff9e35c498e1c0 x1874672086060032/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830485 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830460.936592:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e1c0 x1874672086060032/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830485 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830460.936602:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830460.936606:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830460.936610:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672086060032:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1787830460.936633:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086060160:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830460.936657:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830460.936669:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830460/real 1787830460] req@ffff9e35c498c780 x1874672086060160/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830485 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830460.936759:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c780 x1874672086060160/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830485 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830460.936769:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830460.936893:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830460.936897:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086060160:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1787830460.937274:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830460.937279:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787830460.937281:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830460.937286:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3896 00000100:00080000:2.0:1787830460.937292:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830460.937297:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830460.937304:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830460.937328:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c3c0 x1874672086060288/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830460.937449:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830460.937452:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787830460.937454:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787830460.937496:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086060288:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1787830460.937536:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3896 00000100:00080000:2.0:1787830460.937541:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830460.937550:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830460.937556:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:2.0:1787830460.937568:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f840 x1874672086060416/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830460.937577:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830460.937581:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787830460.937582:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1787830460.937588:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1787830460.937588:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3896 00000100:00100000:0.0:1787830460.937606:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830460/real 1787830460] req@ffff9e35c498c3c0 x1874672086060288/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830485 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830460.937619:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c3c0 x1874672086060288/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830485 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830460.937627:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830460.937631:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830460.937634:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086060288:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830460.937648:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086060416:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830460.937667:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830460.937677:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830460/real 1787830460] req@ffff9e35c498f840 x1874672086060416/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830485 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830460.937684:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830460.937690:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830460.937694:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:2.0:1787830460.937706:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498fc00 x1874672086060544/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830460.937716:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830460.937719:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787830460.937721:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830460.937724:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3896 00000100:00080000:2.0:1787830460.937727:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830460.937730:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830460.937734:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830460.937745:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e580 x1874672086060672/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830460.937755:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830460.937758:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787830460.937759:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787830460.938180:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f840 x1874672086060416/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830485 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830460.938197:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830460.938201:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830460.938205:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086060416:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830460.938220:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086060544:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830460.938245:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830460.938257:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830460/real 1787830460] req@ffff9e35c498fc00 x1874672086060544/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830485 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830460.938265:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3896 00000100:00080000:2.0:1787830460.938271:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787830460.938274:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498fc00 x1874672086060544/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830485 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830460.938277:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830460.938283:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1787830460.938291:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830460.938297:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1787830460.938300:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f840 x1874672086060800/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830460.938305:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086060544:192.168.203.141@tcp:8:kworker.0 00000100:00080000:2.0:1787830460.938311:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787830460.938317:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086060672:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1787830460.938317:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000100:0.0:1787830460.938340:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830460.938351:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830460/real 1787830460] req@ffff9e35c498e580 x1874672086060672/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830485 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830460.938362:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e580 x1874672086060672/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830485 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830460.938369:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830460.938372:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830460.938375:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086060672:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1787830460.938386:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787830460.938390:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086060800:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1787830460.938395:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3896 00000100:00080000:2.0:1787830460.938401:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830460.938405:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830460.938412:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000800:00000100:0.0:1787830460.938416:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787830460.938429:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c3c0 x1874672086060928/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830460.938431:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830460/real 1787830460] req@ffff9e35c498f840 x1874672086060800/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830485 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830460.938455:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787830460.938460:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f840 x1874672086060800/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830485 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830460.938461:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787830460.938464:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830460.938472:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3896 00000100:00080000:0.0:1787830460.938475:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830460.938477:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830460.938481:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:2.0:1787830460.938486:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787830460.938489:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086060800:192.168.203.141@tcp:38:kworker.0 00000008:00100000:2.0:1787830460.938493:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:0.0:1787830460.938502:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086060928:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787830460.938509:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c780 x1874672086061056/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787830460.938584:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830460.938597:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830460/real 1787830460] req@ffff9e35c498c3c0 x1874672086060928/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830485 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830460.938608:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c3c0 x1874672086060928/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830485 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830460.938616:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830460.938619:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830460.938622:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086060928:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830460.938632:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086061056:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830460.938648:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830460.938658:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830460/real 1787830460] req@ffff9e35c498c780 x1874672086061056/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830485 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830460.938669:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c780 x1874672086061056/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830485 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830460.938676:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830460.938679:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830460.938681:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086061056:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830461.063922:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830461.127910:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787830461.127969:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830461.129153:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787830462.087914:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830462.151935:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787830462.151972:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830462.153366:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787830463.111973:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830463.175932:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787830463.175933:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787830463.176254:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830464.137097:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830464.199931:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787830464.199931:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787830464.199985:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830465.159959:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830465.223936:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787830465.223988:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830465.224134:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00080000:2.0:1787830466.055942:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830466.055953:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787830466.055957:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830466.055967:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3901 00000100:00080000:2.0:1787830466.055978:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830466.055984:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787830466.056018:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e1c0 x1874672086061184/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830466.056063:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830466.056067:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787830466.056068:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830466.056072:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3901 00000100:00080000:2.0:1787830466.056076:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830466.056079:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830466.056088:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830466.056169:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cf00 x1874672086061312/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830466.056175:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672086061184:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1787830466.056185:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830466.056191:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787830466.056193:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830466.056198:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3901 00000100:00080000:2.0:1787830466.056202:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830466.056205:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830466.056210:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830466.056221:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f0c0 x1874672086061440/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830466.056230:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830466.056233:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787830466.056234:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1787830466.056241:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1787830466.056247:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3901 00000100:00080000:2.0:1787830466.056253:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830466.056257:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830466.056262:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:2.0:1787830466.056273:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f480 x1874672086061568/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830466.056282:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787830466.056286:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830466/real 1787830466] req@ffff9e35c498e1c0 x1874672086061184/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830491 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830466.056287:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787830466.056290:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830466.056297:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3901 00000100:00080000:2.0:1787830466.056302:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830466.056306:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830466.056311:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:0.0:1787830466.056313:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e1c0 x1874672086061184/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830491 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830466.056322:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498de00 x1874672086061696/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830466.056326:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830466.056336:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:2.0:1787830466.056338:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787830466.056344:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672086061184:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1787830466.056344:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787830466.056346:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830466.056354:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3901 00000100:00080000:2.0:1787830466.056358:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830466.056360:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830466.056364:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830466.056376:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086061312:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787830466.056377:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cb40 x1874672086061824/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830466.056390:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830466.056393:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787830466.056395:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830466.056399:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3901 00000100:00080000:2.0:1787830466.056402:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830466.056406:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:0.0:1787830466.056410:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:2.0:1787830466.056411:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830466.056425:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498d680 x1874672086061952/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830466.056430:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830466/real 1787830466] req@ffff9e35c498cf00 x1874672086061312/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830491 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830466.056437:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830466.056443:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787830466.056445:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787830466.056446:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498cf00 x1874672086061312/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830491 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830466.056454:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830466.056456:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830466.056458:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086061312:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1787830466.056462:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3901 00000100:00100000:0.0:1787830466.056466:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086061440:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1787830466.056467:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830466.056474:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830466.056480:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000800:00000100:0.0:1787830466.056488:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787830466.056495:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e940 x1874672086062080/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830466.056498:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830466/real 1787830466] req@ffff9e35c498f0c0 x1874672086061440/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830491 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830466.056516:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830466.056520:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787830466.056524:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787830466.056525:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f0c0 x1874672086061440/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830491 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830466.056534:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3901 00000100:00080000:0.0:1787830466.056537:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830466.056538:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830466.056546:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:2.0:1787830466.056546:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830466.056554:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:0.0:1787830466.056555:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086061440:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830466.056567:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086061568:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1787830466.056568:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c000 x1874672086062208/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787830466.056595:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830466.056607:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830466/real 1787830466] req@ffff9e35c498f480 x1874672086061568/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830491 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830466.056617:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f480 x1874672086061568/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830491 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830466.056625:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830466.056628:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830466.056631:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086061568:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830466.056637:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086061696:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830466.056655:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830466.056664:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830466/real 1787830466] req@ffff9e35c498de00 x1874672086061696/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830491 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830466.056675:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498de00 x1874672086061696/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830491 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830466.056684:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830466.056688:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830466.056691:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086061696:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830466.056712:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086061824:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830466.056735:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830466.056748:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830466/real 1787830466] req@ffff9e35c498cb40 x1874672086061824/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830491 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830466.056760:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498cb40 x1874672086061824/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830491 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830466.056768:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830466.056771:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830466.056774:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086061824:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830466.056781:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086061952:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830466.056823:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830466.056839:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830466/real 1787830466] req@ffff9e35c498d680 x1874672086061952/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830491 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830466.056850:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498d680 x1874672086061952/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830491 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830466.056859:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830466.056863:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830466.056866:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086061952:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830466.056872:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086062080:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830466.056889:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830466.056896:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830466/real 1787830466] req@ffff9e35c498e940 x1874672086062080/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830491 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830466.056905:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e940 x1874672086062080/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830491 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830466.056911:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830466.056913:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830466.056915:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086062080:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830466.056918:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086062208:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830466.056928:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830466.056935:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830466/real 1787830466] req@ffff9e35c498c000 x1874672086062208/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830491 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830466.056941:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c000 x1874672086062208/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830491 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830466.056947:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830466.056951:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830466.056953:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086062208:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830466.184142:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830466.247919:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787830466.247947:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787830466.248331:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830467.207920:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830467.271919:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787830467.272085:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787830467.275224:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787830468.231966:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830468.295953:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787830468.296056:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830468.296090:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787830469.255994:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830469.319923:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787830469.319939:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787830469.319968:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830470.279924:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830470.343899:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787830470.343899:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787830470.343938:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787830471.175922:0:27971:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830471.175931:0:27971:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787830471.175933:0:27971:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830471.175941:0:27971:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3906 00000100:00080000:3.0:1787830471.175949:0:27971:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830471.175954:0:27971:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787830471.175999:0:27971:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b480 x1874672086062336/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830471.176024:0:27971:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830471.176028:0:27971:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787830471.176029:0:27971:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830471.176032:0:27971:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3906 00000100:00080000:3.0:1787830471.176035:0:27971:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830471.176038:0:27971:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830471.176045:0:27971:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830471.176055:0:27971:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c64992c0 x1874672086062464/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830471.176065:0:27971:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830471.176069:0:27971:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787830471.176071:0:27971:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830471.176076:0:27971:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3906 00000100:00080000:3.0:1787830471.176079:0:27971:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787830471.176080:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672086062336:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1787830471.176084:0:27971:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830471.176092:0:27971:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830471.176102:0:27971:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649ad00 x1874672086062592/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830471.176110:0:27971:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830471.176112:0:27971:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787830471.176114:0:27971:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830471.176117:0:27971:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3906 00000100:00080000:3.0:1787830471.176121:0:27971:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830471.176124:0:27971:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830471.176129:0:27971:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:3.0:1787830471.176141:0:27971:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b840 x1874672086062720/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787830471.176151:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1787830471.176151:0:27971:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830471.176157:0:27971:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787830471.176159:0:27971:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830471.176164:0:27971:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3906 00000100:00080000:3.0:1787830471.176168:0:27971:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830471.176171:0:27971:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830471.176175:0:27971:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:3.0:1787830471.176185:0:27971:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a1c0 x1874672086062848/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830471.176193:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830471/real 1787830471] req@ffff9e35c649b480 x1874672086062336/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830496 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830471.176195:0:27971:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830471.176198:0:27971:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787830471.176199:0:27971:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830471.176204:0:27971:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3906 00000100:00080000:3.0:1787830471.176209:0:27971:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830471.176215:0:27971:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830471.176222:0:27971:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830471.176229:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b480 x1874672086062336/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830496 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787830471.176234:0:27971:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498000 x1874672086062976/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830471.176255:0:27971:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830471.176257:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787830471.176258:0:27971:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787830471.176260:0:27971:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830471.176266:0:27971:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3906 00000100:00080000:0.0:1787830471.176267:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:3.0:1787830471.176271:0:27971:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787830471.176276:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672086062336:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1787830471.176279:0:27971:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830471.176286:0:27971:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830471.176296:0:27971:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c64983c0 x1874672086063104/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830471.176304:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c64992c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086062464:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1787830471.176307:0:27971:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830471.176311:0:27971:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787830471.176313:0:27971:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830471.176318:0:27971:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3906 00000100:00080000:3.0:1787830471.176322:0:27971:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1787830471.176326:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1787830471.176328:0:27971:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830471.176334:0:27971:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:0.0:1787830471.176343:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830471/real 1787830471] req@ffff9e35c64992c0 x1874672086062464/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830496 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787830471.176348:0:27971:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b480 x1874672086063232/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830471.176368:0:27971:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787830471.176373:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c64992c0 x1874672086062464/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830496 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830471.176373:0:27971:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787830471.176377:0:27971:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830471.176384:0:27971:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3906 00000100:00080000:3.0:1787830471.176391:0:27971:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830471.176392:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787830471.176396:0:27971:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:0.0:1787830471.176400:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000008:00100000:3.0:1787830471.176403:0:27971:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:0.0:1787830471.176406:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c64992c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086062464:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830471.176415:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086062592:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1787830471.176417:0:27971:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649bc00 x1874672086063360/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787830471.176442:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830471.176456:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830471/real 1787830471] req@ffff9e35c649ad00 x1874672086062592/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830496 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830471.176468:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649ad00 x1874672086062592/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830496 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830471.176476:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830471.176479:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830471.176482:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086062592:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830471.176488:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086062720:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830471.176504:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830471.176512:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830471/real 1787830471] req@ffff9e35c649b840 x1874672086062720/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830496 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830471.176518:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b840 x1874672086062720/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830496 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830471.176531:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830471.176534:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830471.176537:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086062720:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830471.176542:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086062848:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830471.176559:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830471.176569:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830471/real 1787830471] req@ffff9e35c649a1c0 x1874672086062848/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830496 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830471.176579:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a1c0 x1874672086062848/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830496 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830471.176584:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830471.176586:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830471.176588:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086062848:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830471.176592:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086062976:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830471.176604:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830471.176611:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830471/real 1787830471] req@ffff9e35c6498000 x1874672086062976/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830496 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830471.176618:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498000 x1874672086062976/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830496 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830471.176622:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830471.176623:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830471.176625:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086062976:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830471.176638:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c64983c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086063104:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830471.176648:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830471.176653:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830471/real 1787830471] req@ffff9e35c64983c0 x1874672086063104/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830496 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830471.176660:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c64983c0 x1874672086063104/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830496 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830471.176667:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830471.176670:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830471.176673:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c64983c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086063104:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830471.176678:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086063232:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830471.176695:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830471.176705:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830471/real 1787830471] req@ffff9e35c649b480 x1874672086063232/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830496 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830471.176715:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b480 x1874672086063232/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830496 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830471.176722:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830471.176725:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830471.176728:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086063232:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830471.176733:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086063360:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830471.176748:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830471.176757:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830471/real 1787830471] req@ffff9e35c649bc00 x1874672086063360/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830496 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830471.176767:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649bc00 x1874672086063360/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830496 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830471.176775:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830471.176778:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830471.176781:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086063360:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830471.303928:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830471.367922:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787830471.367922:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787830471.368199:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830472.327964:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830472.393062:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787830472.393411:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830472.394324:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787830473.351955:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830473.415933:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787830473.416043:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787830473.416151:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830474.376228:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830474.439925:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787830474.439987:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787830474.441835:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787830475.399928:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830475.463990:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787830475.463990:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787830475.464127:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787830476.296307:0:27971:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830476.296321:0:27971:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787830476.296325:0:27971:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830476.296335:0:27971:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3911 00000100:00080000:0.0:1787830476.296346:0:27971:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830476.296353:0:27971:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787830476.296390:0:27971:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a1c0 x1874672086063488/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787830476.296466:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672086063488:192.168.203.141@tcp:250:kworker.0 00000800:00000100:1.0:1787830476.296524:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787830476.296716:0:27971:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830476.296720:0:27971:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787830476.296723:0:27971:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830476.296728:0:27971:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3911 00000100:00080000:0.0:1787830476.296732:0:27971:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830476.296736:0:27971:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830476.296747:0:27971:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830476.296761:0:27971:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7192c0 x1874672086063616/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787830476.296838:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830476/real 1787830476] req@ffff9e35da71a1c0 x1874672086063488/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830501 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830476.296856:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a1c0 x1874672086063488/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830501 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830476.296865:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830476.296869:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830476.296873:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672086063488:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1787830476.296894:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086063616:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787830476.296916:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830476.296929:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830476/real 1787830476] req@ffff9e35da7192c0 x1874672086063616/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830501 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830476.296939:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7192c0 x1874672086063616/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830501 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830476.296947:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830476.297072:0:27971:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830476.297076:0:27971:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787830476.297078:0:27971:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830476.297083:0:27971:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3911 00000100:00080000:0.0:1787830476.297087:0:27971:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830476.297091:0:27971:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830476.297096:0:27971:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830476.297109:0:27971:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a580 x1874672086063744/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830476.297120:0:27971:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830476.297123:0:27971:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787830476.297124:0:27971:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830476.297128:0:27971:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3911 00000100:00080000:0.0:1787830476.297131:0:27971:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830476.297135:0:27971:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830476.297140:0:27971:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:0.0:1787830476.297150:0:27971:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718b40 x1874672086063872/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830476.297158:0:27971:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830476.297161:0:27971:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787830476.297163:0:27971:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830476.297167:0:27971:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3911 00000100:00080000:0.0:1787830476.297170:0:27971:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830476.297173:0:27971:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830476.297178:0:27971:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:0.0:1787830476.297188:0:27971:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7183c0 x1874672086064000/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830476.297196:0:27971:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830476.297199:0:27971:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787830476.297201:0:27971:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830476.297204:0:27971:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3911 00000100:00080000:0.0:1787830476.297208:0:27971:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830476.297211:0:27971:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830476.297215:0:27971:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830476.297225:0:27971:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718000 x1874672086064128/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830476.297234:0:27971:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830476.297237:0:27971:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787830476.297238:0:27971:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830476.297242:0:27971:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3911 00000100:00080000:0.0:1787830476.297245:0:27971:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830476.297248:0:27971:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830476.297251:0:27971:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830476.297261:0:27971:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da719a40 x1874672086064256/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830476.297270:0:27971:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830476.297273:0:27971:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787830476.297275:0:27971:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830476.297281:0:27971:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3911 00000100:00080000:1.0:1787830476.297282:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:0.0:1787830476.297285:0:27971:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830476.297292:0:27971:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787830476.297292:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086063616:192.168.203.141@tcp:38:kworker.0 00000008:00100000:0.0:1787830476.297300:0:27971:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:0.0:1787830476.297312:0:27971:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718780 x1874672086064384/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787830476.297315:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086063744:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787830476.297329:0:27971:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830476.297333:0:27971:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787830476.297335:0:27971:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830476.297339:0:27971:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3911 00000100:00080000:0.0:1787830476.297343:0:27971:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1787830476.297345:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787830476.297351:0:27971:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830476.297356:0:27971:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:1.0:1787830476.297357:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830476/real 1787830476] req@ffff9e35da71a580 x1874672086063744/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830501 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830476.297375:0:27971:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a940 x1874672086064512/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787830476.297375:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672086063744/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830501 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830476.297403:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830476.297407:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830476.297410:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086063744:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787830476.297415:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086063872:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787830476.297433:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830476.297443:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830476/real 1787830476] req@ffff9e35da718b40 x1874672086063872/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830501 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830476.297454:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718b40 x1874672086063872/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830501 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830476.297463:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830476.297466:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830476.297470:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086063872:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830476.297475:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086064000:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787830476.297493:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830476.297503:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830476/real 1787830476] req@ffff9e35da7183c0 x1874672086064000/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830501 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830476.297514:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672086064000/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830501 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830476.297521:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830476.297524:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830476.297526:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086064000:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830476.297529:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086064128:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787830476.297542:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830476.297550:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830476/real 1787830476] req@ffff9e35da718000 x1874672086064128/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830501 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830476.297558:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718000 x1874672086064128/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830501 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830476.297563:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830476.297566:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830476.297570:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086064128:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787830476.297575:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086064256:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787830476.297591:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830476.297602:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830476/real 1787830476] req@ffff9e35da719a40 x1874672086064256/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830501 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830476.297613:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672086064256/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830501 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830476.297620:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830476.297623:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830476.297626:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086064256:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787830476.297644:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086064384:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787830476.297663:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830476.297673:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830476/real 1787830476] req@ffff9e35da718780 x1874672086064384/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830501 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830476.297685:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718780 x1874672086064384/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830501 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830476.297692:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830476.297695:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830476.297698:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086064384:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830476.297704:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086064512:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787830476.297721:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830476.297739:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830476/real 1787830476] req@ffff9e35da71a940 x1874672086064512/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830501 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830476.297751:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672086064512/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830501 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830476.297759:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830476.297762:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830476.297765:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086064512:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830476.424180:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830476.487925:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787830476.488037:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830476.488881:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787830477.448010:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830477.511933:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787830477.512007:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830477.512090:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787830478.471928:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830478.535884:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787830478.535919:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787830478.536010:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830479.495903:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787830479.559994:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830479.560026:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787830479.560052:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787830480.519949:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830480.583912:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787830480.583984:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830480.584097:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00080000:3.0:1787830481.416122:0:27971:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830481.416133:0:27971:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787830481.416137:0:27971:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830481.416148:0:27971:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3916 00000100:00080000:3.0:1787830481.416158:0:27971:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830481.416165:0:27971:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787830481.416207:0:27971:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498780 x1874672086064640/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830481.416239:0:27971:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830481.416243:0:27971:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787830481.416244:0:27971:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830481.416248:0:27971:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3916 00000100:00080000:3.0:1787830481.416252:0:27971:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830481.416255:0:27971:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830481.416267:0:27971:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830481.416278:0:27971:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499680 x1874672086064768/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830481.416287:0:27971:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830481.416290:0:27971:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787830481.416292:0:27971:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830481.416295:0:27971:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3916 00000100:00080000:3.0:1787830481.416298:0:27971:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830481.416302:0:27971:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830481.416306:0:27971:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830481.416316:0:27971:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a940 x1874672086064896/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830481.416324:0:27971:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830481.416327:0:27971:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787830481.416328:0:27971:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830481.416331:0:27971:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3916 00000100:00080000:3.0:1787830481.416333:0:27971:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830481.416336:0:27971:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830481.416339:0:27971:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:3.0:1787830481.416346:0:27971:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b0c0 x1874672086065024/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830481.416351:0:27971:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830481.416353:0:27971:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787830481.416354:0:27971:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830481.416357:0:27971:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3916 00000100:00080000:3.0:1787830481.416359:0:27971:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830481.416362:0:27971:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830481.416365:0:27971:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:3.0:1787830481.416374:0:27971:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a580 x1874672086065152/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830481.416382:0:27971:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830481.416385:0:27971:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787830481.416386:0:27971:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830481.416390:0:27971:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3916 00000100:00080000:3.0:1787830481.416393:0:27971:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830481.416396:0:27971:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830481.416400:0:27971:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830481.416412:0:27971:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499e00 x1874672086065280/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830481.416419:0:27971:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830481.416422:0:27971:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787830481.416424:0:27971:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830481.416427:0:27971:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3916 00000100:00080000:3.0:1787830481.416430:0:27971:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830481.416434:0:27971:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830481.416438:0:27971:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830481.416447:0:27971:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498f00 x1874672086065408/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830481.416455:0:27971:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830481.416457:0:27971:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787830481.416458:0:27971:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830481.416460:0:27971:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3916 00000100:00080000:3.0:1787830481.416462:0:27971:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830481.416465:0:27971:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830481.416469:0:27971:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:3.0:1787830481.416478:0:27971:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498b40 x1874672086065536/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830481.416483:0:27971:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830481.416485:0:27971:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787830481.416486:0:27971:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830481.416489:0:27971:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3916 00000100:00080000:3.0:1787830481.416492:0:27971:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830481.416496:0:27971:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830481.416500:0:27971:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:3.0:1787830481.416509:0:27971:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499a40 x1874672086065664/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787830481.416593:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672086064640:192.168.203.141@tcp:250:kworker.0 00000800:00000100:1.0:1787830481.416650:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830481.416687:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830481/real 1787830481] req@ffff9e35c6498780 x1874672086064640/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830506 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830481.416701:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498780 x1874672086064640/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830506 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830481.416711:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830481.416714:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830481.416718:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672086064640:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1787830481.416724:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086064768:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787830481.416743:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830481.416755:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830481/real 1787830481] req@ffff9e35c6499680 x1874672086064768/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830506 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830481.416765:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499680 x1874672086064768/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830506 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830481.416774:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830481.416782:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830481.416785:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086064768:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787830481.416791:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086064896:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787830481.416841:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830481.416852:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830481/real 1787830481] req@ffff9e35c649a940 x1874672086064896/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830506 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830481.416863:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a940 x1874672086064896/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830506 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830481.416871:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830481.416874:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830481.416877:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086064896:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787830481.416882:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086065024:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787830481.416901:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830481.416912:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830481/real 1787830481] req@ffff9e35c649b0c0 x1874672086065024/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830506 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830481.416935:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b0c0 x1874672086065024/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830506 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830481.416943:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830481.416946:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830481.416950:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086065024:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830481.416955:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086065152:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787830481.416972:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830481.416982:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830481/real 1787830481] req@ffff9e35c649a580 x1874672086065152/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830506 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830481.416993:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a580 x1874672086065152/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830506 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830481.417002:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830481.417005:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830481.417008:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086065152:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830481.417013:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086065280:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787830481.417030:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830481.417040:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830481/real 1787830481] req@ffff9e35c6499e00 x1874672086065280/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830506 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830481.417049:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499e00 x1874672086065280/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830506 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830481.417057:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830481.417060:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830481.417063:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086065280:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787830481.417067:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086065408:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787830481.417084:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830481.417094:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830481/real 1787830481] req@ffff9e35c6498f00 x1874672086065408/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830506 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830481.417104:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498f00 x1874672086065408/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830506 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830481.417112:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830481.417114:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830481.417117:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086065408:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787830481.417122:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086065536:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787830481.417141:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830481.417151:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830481/real 1787830481] req@ffff9e35c6498b40 x1874672086065536/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830506 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830481.417162:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498b40 x1874672086065536/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830506 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830481.417170:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830481.417173:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830481.417177:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086065536:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830481.417182:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086065664:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787830481.417200:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830481.417210:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830481/real 1787830481] req@ffff9e35c6499a40 x1874672086065664/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830506 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830481.417220:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499a40 x1874672086065664/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830506 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830481.417227:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830481.417230:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830481.417234:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086065664:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830481.544121:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830481.607944:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787830481.607994:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830481.608036:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787830482.569021:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830482.631931:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787830482.631969:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787830482.632043:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830483.591942:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830483.655942:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787830483.655943:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787830483.656039:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830484.615947:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:1.0:1787830484.679941:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787830484.680249:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787830484.681942:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830485.639941:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787830485.703881:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787830485.704028:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787830485.704103:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787830486.537295:0:27971:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830486.537305:0:27971:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787830486.537309:0:27971:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830486.537319:0:27971:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3921 00000100:00080000:0.0:1787830486.537329:0:27971:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830486.537335:0:27971:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787830486.537368:0:27971:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718f00 x1874672086065792/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830486.537393:0:27971:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830486.537397:0:27971:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787830486.537398:0:27971:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830486.537402:0:27971:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3921 00000100:00080000:0.0:1787830486.537406:0:27971:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830486.537409:0:27971:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830486.537419:0:27971:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830486.537436:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672086065792:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1787830486.537445:0:27971:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a940 x1874672086065920/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830486.537456:0:27971:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830486.537459:0:27971:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787830486.537461:0:27971:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830486.537466:0:27971:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3921 00000100:00080000:0.0:1787830486.537483:0:27971:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830486.537487:0:27971:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:1.0:1787830486.537493:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:0.0:1787830486.537495:0:27971:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830486.537510:0:27971:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718780 x1874672086066048/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830486.537519:0:27971:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830486.537523:0:27971:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787830486.537525:0:27971:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830486.537529:0:27971:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3921 00000100:00080000:0.0:1787830486.537533:0:27971:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1787830486.537537:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830486/real 1787830486] req@ffff9e35da718f00 x1874672086065792/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830511 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830486.537541:0:27971:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830486.537549:0:27971:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:1.0:1787830486.537562:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718f00 x1874672086065792/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830511 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830486.537565:0:27971:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da719a40 x1874672086066176/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830486.537587:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830486.537589:0:27971:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830486.537593:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:0.0:1787830486.537595:0:27971:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787830486.537598:0:27971:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787830486.537600:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672086065792:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1787830486.537606:0:27971:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3921 00000100:00080000:0.0:1787830486.537611:0:27971:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830486.537615:0:27971:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830486.537619:0:27971:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:1.0:1787830486.537628:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086065920:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830486.537633:0:27971:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718000 x1874672086066304/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830486.537646:0:27971:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830486.537649:0:27971:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787830486.537652:0:27971:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1787830486.537656:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787830486.537660:0:27971:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3921 00000100:00080000:0.0:1787830486.537666:0:27971:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830486.537670:0:27971:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787830486.537673:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830486/real 1787830486] req@ffff9e35da71a940 x1874672086065920/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830511 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1787830486.537676:0:27971:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830486.537694:0:27971:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7183c0 x1874672086066432/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787830486.537696:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a940 x1874672086065920/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830511 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830486.537720:0:27971:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830486.537721:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830486.537727:0:27971:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787830486.537729:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:0.0:1787830486.537731:0:27971:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787830486.537736:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086065920:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787830486.537738:0:27971:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3921 00000100:00080000:0.0:1787830486.537745:0:27971:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1787830486.537750:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086066048:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787830486.537753:0:27971:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830486.537761:0:27971:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830486.537774:0:27971:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da718b40 x1874672086066560/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787830486.537780:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1787830486.537790:0:27971:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830486.537794:0:27971:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:1.0:1787830486.537795:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830486/real 1787830486] req@ffff9e35da718780 x1874672086066048/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830511 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830486.537825:0:27971:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830486.537832:0:27971:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3921 00000100:00100000:1.0:1787830486.537834:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718780 x1874672086066048/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830511 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830486.537837:0:27971:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830486.537844:0:27971:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830486.537853:0:27971:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00080000:1.0:1787830486.537859:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830486.537864:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830486.537869:0:27971:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da71a580 x1874672086066688/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787830486.537872:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086066048:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1787830486.537896:0:27971:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830486.537901:0:27971:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:1.0:1787830486.537901:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086066176:192.168.203.141@tcp:8:kworker.0 00000100:00080000:0.0:1787830486.537905:0:27971:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830486.537912:0:27971:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3921 00000100:00080000:0.0:1787830486.537916:0:27971:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830486.537920:0:27971:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1787830486.537925:0:27971:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000800:00000100:1.0:1787830486.537930:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830486.537942:0:27971:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da7192c0 x1874672086066816/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787830486.537946:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830486/real 1787830486] req@ffff9e35da719a40 x1874672086066176/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830511 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830486.537978:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da719a40 x1874672086066176/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830511 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830486.537987:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830486.537990:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830486.537993:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da719a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086066176:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830486.538009:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086066304:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787830486.538026:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830486.538037:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830486/real 1787830486] req@ffff9e35da718000 x1874672086066304/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830511 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830486.538047:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718000 x1874672086066304/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830511 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830486.538190:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830486.538194:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830486.538197:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086066304:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830486.538203:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086066432:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787830486.538226:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830486.538238:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830486/real 1787830486] req@ffff9e35da7183c0 x1874672086066432/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830511 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830486.538249:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7183c0 x1874672086066432/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830511 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830486.538257:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830486.538260:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830486.538264:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7183c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086066432:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787830486.538269:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086066560:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787830486.538286:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830486.538296:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830486/real 1787830486] req@ffff9e35da718b40 x1874672086066560/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830511 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830486.538306:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da718b40 x1874672086066560/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830511 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830486.538313:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830486.538316:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830486.538318:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da718b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086066560:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787830486.538323:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086066688:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787830486.538337:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830486.538345:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830486/real 1787830486] req@ffff9e35da71a580 x1874672086066688/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830511 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830486.538354:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da71a580 x1874672086066688/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830511 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830486.538361:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830486.538364:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830486.538366:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da71a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086066688:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830486.538371:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086066816:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787830486.538385:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830486.538393:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830486/real 1787830486] req@ffff9e35da7192c0 x1874672086066816/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830511 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830486.538402:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da7192c0 x1874672086066816/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830511 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830486.538409:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830486.538412:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830486.538414:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da7192c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086066816:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830486.664063:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787830486.727956:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787830486.728015:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830486.728384:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787830487.687967:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787830487.751941:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787830487.751970:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787830487.751990:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830488.712110:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830488.775934:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787830488.775976:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830488.777760:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787830489.735988:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830489.799952:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787830489.799975:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787830489.800085:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787830490.759916:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830490.824252:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787830490.824313:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787830490.824534:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787830491.655982:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830491.655992:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787830491.655997:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830491.656008:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3927 00000100:00080000:3.0:1787830491.656018:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830491.656024:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787830491.656060:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499a40 x1874672086066944/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830491.656089:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830491.656093:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787830491.656094:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830491.656103:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3927 00000100:00080000:3.0:1787830491.656107:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830491.656111:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830491.656121:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830491.656134:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498b40 x1874672086067072/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830491.656143:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830491.656146:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787830491.656148:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830491.656152:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3927 00000100:00080000:3.0:1787830491.656155:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830491.656160:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787830491.656163:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672086066944:192.168.203.141@tcp:250:kworker.0 00000008:00100000:3.0:1787830491.656166:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830491.656206:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498f00 x1874672086067200/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830491.656217:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830491.656220:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787830491.656222:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1787830491.656227:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1787830491.656228:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3927 00000100:00080000:3.0:1787830491.656234:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830491.656239:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830491.656244:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:3.0:1787830491.656255:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499e00 x1874672086067328/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830491.656264:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830491.656268:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787830491.656270:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787830491.656273:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830491/real 1787830491] req@ffff9e35c6499a40 x1874672086066944/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830516 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830491.656277:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3927 00000100:00080000:3.0:1787830491.656283:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830491.656291:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830491.656297:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:1.0:1787830491.656300:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499a40 x1874672086066944/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830516 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787830491.656311:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a580 x1874672086067456/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830491.656313:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830491.656321:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:3.0:1787830491.656327:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787830491.656328:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672086066944:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1787830491.656335:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787830491.656338:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787830491.656341:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086067072:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1787830491.656346:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3927 00000100:00080000:3.0:1787830491.656351:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830491.656355:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830491.656360:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1787830491.656369:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787830491.656374:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b0c0 x1874672086067584/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787830491.656385:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830491/real 1787830491] req@ffff9e35c6498b40 x1874672086067072/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830516 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830491.656386:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830491.656393:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787830491.656395:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830491.656398:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3927 00000100:00080000:3.0:1787830491.656402:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830491.656405:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830491.656409:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830491.656419:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a940 x1874672086067712/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830491.656427:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830491.656430:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787830491.656431:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830491.656435:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3927 00000100:00080000:3.0:1787830491.656438:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830491.656441:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830491.656445:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:3.0:1787830491.656454:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499680 x1874672086067840/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830491.656462:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830491.656465:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787830491.656466:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830491.656474:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3927 00000100:00080000:3.0:1787830491.656479:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830491.656485:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787830491.656491:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498b40 x1874672086067072/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830516 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1787830491.656491:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00080000:1.0:1787830491.656506:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1787830491.656507:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498780 x1874672086067968/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830491.656513:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830491.656521:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086067072:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787830491.656560:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086067200:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787830491.656581:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830491.656592:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830491/real 1787830491] req@ffff9e35c6498f00 x1874672086067200/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830516 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830491.656610:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498f00 x1874672086067200/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830516 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830491.656619:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830491.656623:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830491.656626:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086067200:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787830491.656631:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086067328:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787830491.656649:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830491.656659:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830491/real 1787830491] req@ffff9e35c6499e00 x1874672086067328/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830516 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830491.656670:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499e00 x1874672086067328/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830516 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830491.656678:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830491.656681:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830491.656684:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086067328:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830491.656690:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086067456:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787830491.656705:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830491.656715:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830491/real 1787830491] req@ffff9e35c649a580 x1874672086067456/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830516 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830491.656726:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a580 x1874672086067456/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830516 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830491.656741:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830491.656744:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830491.656747:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086067456:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830491.656753:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086067584:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787830491.656769:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830491.656779:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830491/real 1787830491] req@ffff9e35c649b0c0 x1874672086067584/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830516 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830491.656790:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b0c0 x1874672086067584/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830516 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830491.656829:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830491.656833:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830491.656837:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086067584:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787830491.656854:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086067712:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1787830491.656873:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830491.656884:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830491/real 1787830491] req@ffff9e35c649a940 x1874672086067712/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830516 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830491.656895:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a940 x1874672086067712/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830516 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830491.656902:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830491.656904:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830491.656907:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086067712:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787830491.656913:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086067840:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787830491.656931:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830491.656941:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830491/real 1787830491] req@ffff9e35c6499680 x1874672086067840/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830516 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830491.656952:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6499680 x1874672086067840/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830516 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830491.656960:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830491.656964:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830491.656966:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6499680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086067840:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830491.656972:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086067968:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1787830491.656989:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1787830491.656999:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830491/real 1787830491] req@ffff9e35c6498780 x1874672086067968/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830516 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830491.657010:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498780 x1874672086067968/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830516 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830491.657017:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830491.657019:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1787830491.657022:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086067968:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830491.784183:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787830491.848468:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787830491.849134:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787830491.849235:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787830492.808028:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787830492.871946:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787830492.871948:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:0.0:1787830492.871997:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830493.831972:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787830493.895923:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787830493.896030:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787830493.896295:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830494.856153:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787830494.919891:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:3.0:1787830494.920148:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830494.920611:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787830495.883022:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830495.943927:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787830495.944004:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787830495.944219:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787830496.777086:0:27971:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830496.777097:0:27971:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787830496.777101:0:27971:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830496.777111:0:27971:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3932 00000100:00080000:1.0:1787830496.777131:0:27971:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830496.777593:0:27971:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787830496.777629:0:27971:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7480 x1874672086068096/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830496.777664:0:27971:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830496.777668:0:27971:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787830496.777670:0:27971:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830496.777673:0:27971:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3932 00000100:00080000:1.0:1787830496.777678:0:27971:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830496.777681:0:27971:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830496.777690:0:27971:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830496.777702:0:27971:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4780 x1874672086068224/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830496.777712:0:27971:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830496.777715:0:27971:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787830496.777716:0:27971:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830496.777720:0:27971:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3932 00000100:00080000:1.0:1787830496.777723:0:27971:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830496.777726:0:27971:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830496.777730:0:27971:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830496.777739:0:27971:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f61c0 x1874672086068352/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830496.777747:0:27971:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830496.777750:0:27971:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787830496.777751:0:27971:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830496.777755:0:27971:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3932 00000100:00080000:1.0:1787830496.777758:0:27971:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830496.777761:0:27971:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830496.777764:0:27971:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:1.0:1787830496.777772:0:27971:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f43c0 x1874672086068480/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830496.777780:0:27971:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830496.777783:0:27971:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787830496.777785:0:27971:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830496.777788:0:27971:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3932 00000100:00080000:1.0:1787830496.777790:0:27971:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830496.777793:0:27971:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830496.777827:0:27971:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:1.0:1787830496.777839:0:27971:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f7840 x1874672086068608/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830496.777853:0:27971:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830496.777855:0:27971:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787830496.777857:0:27971:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830496.777861:0:27971:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3932 00000100:00080000:1.0:1787830496.777864:0:27971:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830496.777867:0:27971:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830496.777872:0:27971:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830496.777881:0:27971:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6d00 x1874672086068736/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830496.777889:0:27971:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830496.777892:0:27971:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787830496.777893:0:27971:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830496.777897:0:27971:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3932 00000100:00080000:1.0:1787830496.777900:0:27971:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830496.777903:0:27971:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830496.777907:0:27971:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830496.777917:0:27971:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6580 x1874672086068864/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830496.777925:0:27971:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830496.777928:0:27971:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787830496.777930:0:27971:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830496.777933:0:27971:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3932 00000100:00080000:1.0:1787830496.777936:0:27971:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830496.777940:0:27971:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830496.777944:0:27971:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:1.0:1787830496.777951:0:27971:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4f00 x1874672086068992/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830496.777959:0:27971:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830496.777974:0:27971:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787830496.777976:0:27971:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830496.777979:0:27971:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3932 00000100:00080000:1.0:1787830496.777982:0:27971:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830496.777986:0:27971:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830496.777990:0:27971:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:1.0:1787830496.777999:0:27971:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4000 x1874672086069120/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830496.778451:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672086068096:192.168.203.141@tcp:250:kworker.0 00000800:00000100:0.0:1787830496.778507:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830496.778547:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830496/real 1787830496] req@ffff9e35da6f7480 x1874672086068096/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830521 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830496.778563:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7480 x1874672086068096/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830521 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830496.778573:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830496.778577:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830496.778581:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672086068096:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1787830496.778588:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086068224:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830496.778609:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830496.778620:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830496/real 1787830496] req@ffff9e35da6f4780 x1874672086068224/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830521 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830496.778632:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4780 x1874672086068224/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830521 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830496.778640:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830496.778642:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830496.778645:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086068224:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830496.778651:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086068352:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830496.778670:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830496.778681:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830496/real 1787830496] req@ffff9e35da6f61c0 x1874672086068352/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830521 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830496.778692:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f61c0 x1874672086068352/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830521 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830496.778705:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830496.778708:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830496.778711:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f61c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086068352:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830496.778716:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086068480:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830496.778734:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830496.778744:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830496/real 1787830496] req@ffff9e35da6f43c0 x1874672086068480/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830521 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830496.778755:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f43c0 x1874672086068480/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830521 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830496.778763:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830496.778765:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830496.778768:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f43c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086068480:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830496.778774:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086068608:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830496.778791:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830496.778838:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830496/real 1787830496] req@ffff9e35da6f7840 x1874672086068608/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830521 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830496.778850:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f7840 x1874672086068608/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830521 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830496.778858:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830496.778861:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830496.778865:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f7840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086068608:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830496.778871:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086068736:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830496.778890:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830496.778900:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830496/real 1787830496] req@ffff9e35da6f6d00 x1874672086068736/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830521 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830496.778911:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6d00 x1874672086068736/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830521 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830496.778918:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830496.778921:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830496.778924:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086068736:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830496.778929:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086068864:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830496.778946:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830496.778955:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830496/real 1787830496] req@ffff9e35da6f6580 x1874672086068864/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830521 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830496.778976:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6580 x1874672086068864/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830521 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830496.778983:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830496.778985:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830496.778988:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086068864:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830496.778993:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086068992:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830496.779011:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830496.779022:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830496/real 1787830496] req@ffff9e35da6f4f00 x1874672086068992/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830521 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830496.779034:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4f00 x1874672086068992/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830521 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830496.779042:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830496.779045:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830496.779048:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086068992:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830496.780605:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086069120:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830496.780645:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830496.780664:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830496/real 1787830496] req@ffff9e35da6f4000 x1874672086069120/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830521 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830496.780679:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4000 x1874672086069120/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830521 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830496.780688:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830496.780691:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830496.780695:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086069120:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830496.903932:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830496.967927:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787830496.968401:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787830496.968662:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830497.928419:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00010000:00010000:3.0:1787830497.992884:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787830497.993039:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787830497.993102:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787830498.954859:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787830499.015973:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:0.0:1787830499.016095:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:0.0:1787830499.016458:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830499.975927:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830500.039929:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787830500.040125:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787830500.040628:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830500.999926:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830501.064041:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787830501.064163:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787830501.064222:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787830501.898485:0:27971:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830501.898497:0:27971:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787830501.898501:0:27971:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830501.898511:0:27971:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3937 00000100:00080000:2.0:1787830501.898523:0:27971:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830501.898530:0:27971:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1787830501.898567:0:27971:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498ed00 x1874672086069248/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830501.898600:0:27971:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830501.898604:0:27971:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787830501.898605:0:27971:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830501.898609:0:27971:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3937 00000100:00080000:2.0:1787830501.898613:0:27971:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830501.898617:0:27971:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830501.898627:0:27971:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830501.898641:0:27971:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498c000 x1874672086069376/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830501.898652:0:27971:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830501.898654:0:27971:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787830501.898656:0:27971:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830501.898660:0:27971:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3937 00000100:00080000:2.0:1787830501.898664:0:27971:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830501.898667:0:27971:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830501.898672:0:27971:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830501.898683:0:27971:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498e940 x1874672086069504/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830501.898692:0:27971:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830501.898696:0:27971:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787830501.898698:0:27971:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830501.898702:0:27971:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3937 00000100:00080000:2.0:1787830501.898706:0:27971:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830501.898710:0:27971:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830501.898715:0:27971:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:2.0:1787830501.898725:0:27971:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498d680 x1874672086069632/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830501.898732:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672086069248:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1787830501.898739:0:27971:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830501.898744:0:27971:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787830501.898746:0:27971:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830501.898750:0:27971:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3937 00000100:00080000:2.0:1787830501.898754:0:27971:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830501.898758:0:27971:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830501.898763:0:27971:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:2.0:1787830501.898787:0:27971:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cb40 x1874672086069760/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830501.898813:0:27971:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830501.898817:0:27971:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787830501.898818:0:27971:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830501.898823:0:27971:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3937 00000100:00080000:2.0:1787830501.898826:0:27971:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830501.898830:0:27971:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830501.898835:0:27971:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1787830501.898839:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1787830501.898849:0:27971:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498de00 x1874672086069888/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830501.898858:0:27971:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830501.898862:0:27971:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787830501.898863:0:27971:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830501.898867:0:27971:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3937 00000100:00080000:2.0:1787830501.898870:0:27971:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830501.898874:0:27971:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830501.898879:0:27971:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830501.898885:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830501/real 1787830501] req@ffff9e35c498ed00 x1874672086069248/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830526 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830501.898891:0:27971:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f480 x1874672086070016/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830501.898909:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498ed00 x1874672086069248/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830526 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830501.898913:0:27971:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830501.898921:0:27971:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787830501.898925:0:27971:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830501.898929:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830501.898932:0:27971:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3937 00000100:00080000:0.0:1787830501.898937:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:2.0:1787830501.898939:0:27971:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787830501.898944:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672086069248:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1787830501.898947:0:27971:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830501.898955:0:27971:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:0.0:1787830501.898956:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086069376:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1787830501.898970:0:27971:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498f0c0 x1874672086070144/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830501.898979:0:27971:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1787830501.898985:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1787830501.898985:0:27971:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787830501.898988:0:27971:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830501.898995:0:27971:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3937 00000100:00080000:2.0:1787830501.898999:0:27971:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787830501.899003:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830501/real 1787830501] req@ffff9e35c498c000 x1874672086069376/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830526 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830501.899007:0:27971:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1787830501.899016:0:27971:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:0.0:1787830501.899023:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498c000 x1874672086069376/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830526 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830501.899030:0:27971:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c498cf00 x1874672086070272/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830501.899037:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830501.899044:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830501.899049:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086069376:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830501.899055:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086069504:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830501.899076:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830501.899087:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830501/real 1787830501] req@ffff9e35c498e940 x1874672086069504/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830526 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830501.899105:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498e940 x1874672086069504/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830526 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830501.899112:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830501.899115:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830501.899118:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086069504:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830501.899146:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086069632:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830501.899165:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830501.899189:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830501/real 1787830501] req@ffff9e35c498d680 x1874672086069632/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830526 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830501.899200:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498d680 x1874672086069632/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830526 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830501.899208:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830501.899211:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830501.899214:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086069632:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830501.899220:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086069760:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830501.899237:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830501.899248:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830501/real 1787830501] req@ffff9e35c498cb40 x1874672086069760/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830526 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830501.899258:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498cb40 x1874672086069760/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830526 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830501.899271:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830501.899274:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830501.899277:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086069760:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830501.899282:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086069888:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830501.899299:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830501.899308:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830501/real 1787830501] req@ffff9e35c498de00 x1874672086069888/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830526 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830501.899333:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498de00 x1874672086069888/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830526 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830501.899341:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830501.899344:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830501.899348:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086069888:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830501.899353:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086070016:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830501.899371:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830501.899393:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830501/real 1787830501] req@ffff9e35c498f480 x1874672086070016/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830526 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830501.899846:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f480 x1874672086070016/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830526 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830501.899854:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830501.899856:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830501.899860:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086070016:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830501.899867:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086070144:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830501.899889:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830501.899900:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830501/real 1787830501] req@ffff9e35c498f0c0 x1874672086070144/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830526 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830501.899911:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498f0c0 x1874672086070144/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830526 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830501.899919:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830501.899923:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830501.899926:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086070144:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830501.899932:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086070272:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830501.899950:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830501.899960:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830501/real 1787830501] req@ffff9e35c498cf00 x1874672086070272/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830526 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830501.899971:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c498cf00 x1874672086070272/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830526 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830501.899979:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830501.899982:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830501.899985:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c498cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086070272:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830502.023934:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787830502.088116:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:0.0:1787830502.088251:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787830502.088292:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787830503.047932:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:3.0:1787830503.112195:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:2.0:1787830503.112324:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787830503.112395:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830504.071969:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:3.0:1787830504.135945:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787830504.136085:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787830504.136159:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787830505.096064:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787830505.159935:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787830505.160062:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787830505.161896:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787830506.119940:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830506.184004:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:3.0:1787830506.184714:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:1.0:1787830506.185049:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787830507.016407:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830507.016418:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787830507.016422:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830507.016432:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3942 00000100:00080000:3.0:1787830507.016441:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830507.016447:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787830507.016597:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649bc00 x1874672086070400/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830507.016671:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672086070400:192.168.203.141@tcp:250:kworker.0 00000800:00000100:0.0:1787830507.016726:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1787830507.016768:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830507.016772:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787830507.016774:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830507.016778:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3942 00000100:00080000:3.0:1787830507.016782:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830507.016786:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830507.016795:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830507.016837:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b480 x1874672086070528/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830507.016972:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830507/real 1787830507] req@ffff9e35c649bc00 x1874672086070400/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830532 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830507.016989:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649bc00 x1874672086070400/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830532 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830507.016998:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830507.017001:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830507.017005:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672086070400:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1787830507.017026:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086070528:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830507.017046:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830507.017057:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830507/real 1787830507] req@ffff9e35c649b480 x1874672086070528/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830532 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830507.017067:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b480 x1874672086070528/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830532 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830507.017076:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830507.017079:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830507.017081:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086070528:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1787830507.017232:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830507.017236:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787830507.017238:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830507.017245:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3942 00000100:00080000:3.0:1787830507.017249:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830507.017254:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830507.017260:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830507.017274:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c64983c0 x1874672086070656/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830507.017287:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830507.017291:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787830507.017293:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830507.017298:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3942 00000100:00080000:3.0:1787830507.017301:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787830507.017307:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c64983c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086070656:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1787830507.017307:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:0.0:1787830507.017337:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830507.017351:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830507/real 1787830507] req@ffff9e35c64983c0 x1874672086070656/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830532 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830507.017367:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c64983c0 x1874672086070656/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830532 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1787830507.017371:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00080000:0.0:1787830507.017382:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830507.017388:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1787830507.017388:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498000 x1874672086070784/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830507.017394:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c64983c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086070656:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1787830507.017408:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830507.017412:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787830507.017414:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787830507.017417:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086070784:192.168.203.141@tcp:8:kworker.0 00000100:00080000:3.0:1787830507.017421:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3942 00000100:00080000:3.0:1787830507.017427:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830507.017431:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830507.017437:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000800:00000100:0.0:1787830507.017446:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1787830507.017451:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a1c0 x1874672086070912/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830507.017463:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830507/real 1787830507] req@ffff9e35c6498000 x1874672086070784/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830532 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830507.017463:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830507.017467:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787830507.017468:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830507.017473:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3942 00000100:00080000:3.0:1787830507.017476:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830507.017479:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830507.017483:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830507.017491:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b840 x1874672086071040/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830507.017498:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830507.017501:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787830507.017502:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830507.017506:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3942 00000100:00080000:3.0:1787830507.017508:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830507.017512:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830507.017515:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830507.017522:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649ad00 x1874672086071168/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830507.017530:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830507.017533:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787830507.017535:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830507.017538:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3942 00000100:00080000:3.0:1787830507.017540:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830507.017543:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830507.017546:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:3.0:1787830507.017554:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c64992c0 x1874672086071296/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830507.017561:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830507.017564:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787830507.017566:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830507.017569:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3942 00000100:00080000:3.0:1787830507.017572:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830507.017575:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830507.017579:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:3.0:1787830507.017586:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c64983c0 x1874672086071424/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830507.017659:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c6498000 x1874672086070784/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830532 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830507.017669:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830507.017672:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830507.017676:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c6498000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086070784:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830507.017689:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086070912:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830507.017712:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830507.017723:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830507/real 1787830507] req@ffff9e35c649a1c0 x1874672086070912/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830532 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830507.017735:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649a1c0 x1874672086070912/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830532 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830507.017957:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830507.017962:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830507.017965:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086070912:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830507.017972:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086071040:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830507.017993:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830507.018006:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830507/real 1787830507] req@ffff9e35c649b840 x1874672086071040/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830532 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830507.018018:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649b840 x1874672086071040/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830532 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830507.018025:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830507.018027:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830507.018030:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086071040:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830507.018035:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086071168:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830507.018064:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830507.018075:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830507/real 1787830507] req@ffff9e35c649ad00 x1874672086071168/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830532 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830507.018086:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c649ad00 x1874672086071168/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830532 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830507.018094:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830507.018096:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830507.018100:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c649ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086071168:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830507.018105:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c64992c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086071296:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830507.018124:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830507.018134:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830507/real 1787830507] req@ffff9e35c64992c0 x1874672086071296/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830532 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830507.018144:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c64992c0 x1874672086071296/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830532 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830507.018152:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830507.018155:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830507.018158:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c64992c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086071296:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830507.018164:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c64983c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086071424:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830507.018181:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830507.018191:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830507/real 1787830507] req@ffff9e35c64983c0 x1874672086071424/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830532 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830507.018203:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35c64983c0 x1874672086071424/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830532 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830507.018212:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830507.018215:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830507.018218:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35c64983c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086071424:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830507.143938:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830507.207931:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787830507.208563:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787830507.208667:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787830508.168223:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787830508.231925:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:3.0:1787830508.232112:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787830508.232776:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830509.192234:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787830509.255943:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:0.0:1787830509.256050:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787830509.256148:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830510.215993:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787830510.279935:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787830510.279981:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787830510.281184:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830511.239928:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787830511.304038:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787830511.304094:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787830511.304332:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00080000:1.0:1787830512.136556:0:27971:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830512.136569:0:27971:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787830512.136573:0:27971:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830512.136585:0:27971:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3947 00000100:00080000:1.0:1787830512.136596:0:27971:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830512.136604:0:27971:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1787830512.136641:0:27971:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4b40 x1874672086071552/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830512.136718:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672086071552:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1787830512.136732:0:27971:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830512.136735:0:27971:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787830512.136737:0:27971:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830512.136742:0:27971:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3947 00000800:00000100:0.0:1787830512.136778:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787830512.136831:0:27971:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830512.136836:0:27971:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830512.136845:0:27971:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830512.136846:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830512/real 1787830512] req@ffff9e35da6f4b40 x1874672086071552/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830537 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830512.136857:0:27971:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5e00 x1874672086071680/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830512.136864:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4b40 x1874672086071552/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830537 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830512.136872:0:27971:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830512.136875:0:27971:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787830512.136877:0:27971:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830512.136880:0:27971:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3947 00000100:00080000:1.0:1787830512.136884:0:27971:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830512.136887:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830512.136887:0:27971:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:0.0:1787830512.136890:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000008:00100000:1.0:1787830512.136891:0:27971:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830512.136894:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672086071552:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1787830512.136899:0:27971:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f5a40 x1874672086071808/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830512.136906:0:27971:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830512.136909:0:27971:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787830512.136910:0:27971:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787830512.136913:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086071680:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1787830512.136914:0:27971:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3947 00000100:00080000:1.0:1787830512.136917:0:27971:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830512.136920:0:27971:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830512.136924:0:27971:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:1.0:1787830512.136932:0:27971:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f70c0 x1874672086071936/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787830512.136933:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787830512.136941:0:27971:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787830512.136944:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830512/real 1787830512] req@ffff9e35da6f5e00 x1874672086071680/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830537 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830512.136944:0:27971:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787830512.136946:0:27971:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830512.136950:0:27971:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3947 00000100:00080000:1.0:1787830512.136953:0:27971:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787830512.136955:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5e00 x1874672086071680/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830537 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830512.136956:0:27971:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830512.136960:0:27971:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00080000:0.0:1787830512.136964:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830512.136967:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830512.136970:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086071680:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830512.136975:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086071808:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787830512.136990:0:27971:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6940 x1874672086072064/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787830512.136992:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787830512.136999:0:27971:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787830512.137001:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830512/real 1787830512] req@ffff9e35da6f5a40 x1874672086071808/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830537 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830512.137002:0:27971:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787830512.137004:0:27971:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830512.137007:0:27971:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3947 00000100:00080000:1.0:1787830512.137010:0:27971:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830512.137014:0:27971:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1787830512.137018:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f5a40 x1874672086071808/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830537 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1787830512.137022:0:27971:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1787830512.137025:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830512.137027:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830512.137031:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f5a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086071808:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1787830512.137032:0:27971:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4b40 x1874672086072192/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830512.137039:0:27971:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830512.137042:0:27971:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787830512.137044:0:27971:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787830512.137046:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086071936:192.168.203.141@tcp:8:kworker.0 00000100:00080000:1.0:1787830512.137047:0:27971:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3947 00000100:00080000:1.0:1787830512.137050:0:27971:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830512.137053:0:27971:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830512.137057:0:27971:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830512.137066:0:27971:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4000 x1874672086072320/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787830512.137072:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830512.137082:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830512/real 1787830512] req@ffff9e35da6f70c0 x1874672086071936/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830537 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830512.137084:0:27971:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830512.137090:0:27971:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787830512.137092:0:27971:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787830512.137093:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f70c0 x1874672086071936/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830537 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830512.137096:0:27971:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3947 00000100:00080000:1.0:1787830512.137099:0:27971:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830512.137100:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830512.137102:0:27971:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:0.0:1787830512.137103:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830512.137106:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f70c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086071936:192.168.203.141@tcp:8:kworker.0 00000008:00100000:1.0:1787830512.137107:0:27971:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:0.0:1787830512.137111:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086072064:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830512.137116:0:27971:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f4f00 x1874672086072448/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830512.137123:0:27971:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1787830512.137126:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1787830512.137127:0:27971:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787830512.137128:0:27971:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830512.137132:0:27971:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3947 00000100:00080000:1.0:1787830512.137135:0:27971:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830512.137138:0:27971:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1787830512.137142:0:27971:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:1.0:1787830512.137151:0:27971:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35da6f6580 x1874672086072576/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830512.137254:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830512/real 1787830512] req@ffff9e35da6f6940 x1874672086072064/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830537 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830512.137266:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6940 x1874672086072064/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830537 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830512.137280:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830512.137284:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830512.137286:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086072064:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830512.137292:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086072192:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830512.137312:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830512.137322:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830512/real 1787830512] req@ffff9e35da6f4b40 x1874672086072192/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830537 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830512.137332:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4b40 x1874672086072192/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830537 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830512.137339:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830512.137342:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830512.137345:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086072192:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830512.137358:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086072320:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1787830512.137377:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830512.137386:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830512/real 1787830512] req@ffff9e35da6f4000 x1874672086072320/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830537 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830512.137398:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4000 x1874672086072320/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830537 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830512.137405:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830512.137408:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830512.137411:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086072320:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830512.137417:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086072448:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830512.137434:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830512.137443:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830512/real 1787830512] req@ffff9e35da6f4f00 x1874672086072448/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830537 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830512.137455:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f4f00 x1874672086072448/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830537 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830512.137462:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830512.137465:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830512.137467:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f4f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086072448:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830512.137472:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086072576:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1787830512.137487:0:2408:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1787830512.137496:0:2408:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830512/real 1787830512] req@ffff9e35da6f6580 x1874672086072576/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830537 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830512.137505:0:2408:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9e35da6f6580 x1874672086072576/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830537 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830512.137512:0:2408:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830512.137515:0:2408:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1787830512.137518:0:2408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9e35da6f6580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086072576:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830512.263997:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:3.0:1787830512.328013:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787830512.328665:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787830512.329021:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:1.0:1787830513.287974:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:3.0:1787830513.353977:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787830513.354046:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787830513.354108:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:1.0:1787830514.311931:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787830514.375953:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787830514.376225:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787830514.376277:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830515.337110:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787830515.399949:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00000100:00100000:3.0:1787830515.400399:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00010000:00010000:2.0:1787830515.400484:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830516.360018:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:3.0:1787830516.423935:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787830516.423994:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:3.0:1787830516.424004:0:15665:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787830517.256237:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) d88fbf91-ecfb-40ad-b5af-8efd79fdee07->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830517.256249:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787830517.256253:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830517.256263:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3952 00000100:00080000:3.0:1787830517.256273:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830517.256280:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff9e35d02f4800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1787830517.256314:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b480 x1874672086072704/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830517.256345:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830517.256350:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787830517.256352:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8ec9800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830517.256356:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3952 00000100:00080000:3.0:1787830517.256359:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830517.256363:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35d0749800: import ffff9e35c8ec9800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830517.256372:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830517.256384:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649bc00 x1874672086072832/t0(0) o38->lustre-MDT0000-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830517.256394:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830517.256397:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787830517.256398:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f7800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830517.256402:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3952 00000100:00080000:3.0:1787830517.256405:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830517.256409:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35d0749800: import ffff9e35c60f7800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830517.256413:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830517.256421:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498780 x1874672086072960/t0(0) o38->lustre-MDT0001-mdc-ffff9e35d0749800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830517.256429:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830517.256432:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787830517.256433:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c60f1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830517.256436:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3952 00000100:00080000:3.0:1787830517.256439:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830517.256443:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35d0749800: import ffff9e35c60f1000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830517.256448:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 1241088, lost: 0. 00000100:00100000:3.0:1787830517.256460:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499680 x1874672086073088/t0(0) o8->lustre-OST0000-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830517.256468:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d88fbf91-ecfb-40ad-b5af-8efd79fdee07:2408:1874672086072704:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1787830517.256472:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 66d74899-006e-4207-beb4-e14ddbbfbbe2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830517.256477:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787830517.256478:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35d02f9000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830517.256484:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: connect to NID 192.168.203.141@tcp last attempt 3952 00000100:00080000:3.0:1787830517.256488:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830517.256492:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35d0749800: import ffff9e35d02f9000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830517.256496:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 4734976, lost: 0. 00000100:00100000:3.0:1787830517.256507:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a940 x1874672086073216/t0(0) o8->lustre-OST0001-osc-ffff9e35d0749800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830517.256515:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830517.256518:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787830517.256520:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830517.256524:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3952 00000100:00080000:3.0:1787830517.256528:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830517.256531:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9e35e0661800: import ffff9e35c8e00800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830517.256536:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830517.256546:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649b0c0 x1874672086073344/t0(0) o38->lustre-MDT0000-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830517.256579:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086072832:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830517.256618:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086072960:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830517.256641:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086073088:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830517.256661:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:66d74899-006e-4207-beb4-e14ddbbfbbe2:2408:1874672086073216:192.168.203.141@tcp:8:kworker.0 00000100:00080000:3.0:1787830517.257043:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830517.257047:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787830517.257050:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e03800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830517.257056:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3952 00000100:00080000:3.0:1787830517.257060:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830517.257065:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9e35e0661800: import ffff9e35c8e03800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830517.257071:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683204 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830517.257085:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c649a580 x1874672086073472/t0(0) o38->lustre-MDT0001-mdc-ffff9e35e0661800@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830517.257095:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830517.257099:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787830517.257101:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e01800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830517.257105:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3952 00000100:00080000:3.0:1787830517.257108:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830517.257111:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9e35e0661800: import ffff9e35c8e01800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830517.257116:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 7413760, lost: 0. 00000100:00100000:3.0:1787830517.257126:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6499e00 x1874672086073600/t0(0) o8->lustre-OST0000-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830517.257135:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 5a777d8d-8e74-43dd-b307-248963eb234b->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830517.257138:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787830517.257140:0:36:0:(import.c:65:import_set_state_nolock()) ffff9e35c8e00000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787830517.257144:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086073344:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1787830517.257147:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: connect to NID 192.168.203.141@tcp last attempt 3952 00000100:00080000:3.0:1787830517.257154:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830517.257159:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9e35e0661800: import ffff9e35c8e00000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1787830517.257164:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683204 ocd_grant: 602112, lost: 0. 00000100:00100000:3.0:1787830517.257177:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9e35c6498f00 x1874672086073728/t0(0) o8->lustre-OST0001-osc-ffff9e35e0661800@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830517.257184:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c649a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086073472:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1787830517.257209:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6499e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086073600:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1787830517.257228:0:2408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9e35c6498f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:5a777d8d-8e74-43dd-b307-248963eb234b:2408:1874672086073728:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1787830517.386794:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:0.0:1787830517.449291:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds 00000100:00100000:2.0:1787830517.449789:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:1.0:1787830517.453682:0:27501:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830518.408006:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35c7c0f780 going to sleep for 0 seconds 00000100:00100000:2.0:1787830518.471937:0:15664:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e35d1447300 going to sleep for 0 seconds 00010000:00010000:2.0:1787830518.472156:0:6168:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787830518.472445:0:6167:0:(client.c:2715:ptlrpc_set_wait()) set ffff9e36020cc9c0 going to sleep for 0 seconds Debug log: 79446 lines, 79446 kept, 0 dropped, 0 bad.